diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c2b3e08aa38fe..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,306 +0,0 @@ -commands: - print_versions: - description: Version Info - steps: - - run: - name: Version Info - command: | - rustup show - rustc --version - cargo --version - rustup --version - python3 --version - - init_opam: - description: Init Opam - steps: - - run: - name: Init opam - command: | - opam init --compiler=5.1.0 --disable-sandboxing -y - opam install menhir ppxlib -y - - run: - name: OCaml Configuration Info - command: | - eval $(opam env) - ocamlopt.opt -config - - run: - name: Set OCaml envs - command: | - echo 'eval $(opam env)' >> "$BASH_ENV" - - setup_linux_env: - description: Setup env for Linux - steps: - - run: sudo apt-get update - - run: sudo apt-get install libssl-dev cmake clang lld opam libzstd-dev python3-pip ghc - - run: sudo pip3 install conan==1.* - - run: - # the xlarge linux resource class has 8 CPUs, limit the number of jobs to 6 to avoid running out of resources - name: "Set CARGO_BUILD_JOBS=6 to limit the number of CPUs used" - command: echo 'export CARGO_BUILD_JOBS="6"' >> "$BASH_ENV" - - print_versions - - setup_macos_env: - description: Setup env for macOS - steps: - - run: - name: Install Rustup - command: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - run: - name: Increase open file descriptor limit - command: | - # Avoid "too many open files" error. - echo 'sudo launchctl limit maxfiles 9000000 9999999' >> "$BASH_ENV" - echo 'ulimit -Sn 9000000' >> "$BASH_ENV" - - run: - name: Brew install - command: | - # Avoid: "Error: The `brew link` step did not complete - # successfully" (for llvm dependency 'six'). - rm -f '/usr/local/lib/python3.9/site-packages/six.py' - brew install cmake python3 coreutils opam llvm protobuf zstd ghc - # TODO: Remove once non intel macos platform is supported on https://github.com/stepancheg/rust-protoc-bin-vendored/ - echo 'export BUCK2_BUILD_PROTOC=/opt/homebrew/opt/protobuf/bin/protoc' >> "$BASH_ENV" - echo 'export BUCK2_BUILD_PROTOC_INCLUDE=/opt/homebrew/opt/protobuf/include' >> "$BASH_ENV" - - run: sudo pip3 install conan==1.* - - run: - # the xlarge linux resource class has 8 CPUs, limit the number of jobs to 6 to avoid running out of resources - name: "Set CARGO_BUILD_JOBS=6 to limit the number of CPUs used" - command: echo 'export CARGO_BUILD_JOBS="6"' >> "$BASH_ENV" - - run: - name: "Add LLVM to PATH" - command: | - echo 'export PATH=/usr/local/opt/llvm/bin:"$PATH"' >> "$BASH_ENV" - - - print_versions - - setup_windows_env: - description: Setup env for Windows - steps: - - run: - # Use Rust toolchain installed by Rustup and uninstall default one. - name: Install Rustup - command: | - choco uninstall -y rust - choco install -y rustup.install - write-output "[net]`ngit-fetch-with-cli = true" | out-file -append -encoding utf8 $Env:USERPROFILE/.cargo/config.toml - type $Env:USERPROFILE/.cargo/config.toml - - run: - name: Create python3 symlink - command: | - New-Item -ItemType SymbolicLink -Path C:\ProgramData\chocolatey\bin\python3.exe -Target $(Get-Command python).Source - - run: - name: Write Powershell profile - command: | - $psProfileContent = @' - $vsPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -property installationPath - $llvmPath = Join-Path $vsPath "VC\Tools\Llvm\x64\bin" - $env:PATH = "$env:USERPROFILE\.cargo\bin;$llvmPath;" + $env:PATH - $env:TEMP = "$env:USERPROFILE\temp" - $env:TMP = $env:TEMP - '@ - Add-Content "$PsHome\profile.ps1" $psProfileContent - New-Item -ItemType Directory -Path "$env:USERPROFILE\temp" - - print_versions - - setup_reindeer: - description: Install Reindeer - steps: - - run: - name: Install Reindeer - command: | - cargo install --locked --git https://github.com/facebookincubator/reindeer reindeer - reindeer --third-party-dir shim/third-party/rust buckify - - build_debug: - description: Build buck2 binary (debug) - steps: - - run: - name: Build buck2 binary (debug) - command: | - mkdir /tmp/artifacts - cargo build --bin=buck2 -Z unstable-options --out-dir=/tmp/artifacts - - build_release: - description: Build buck2 binary (release) - steps: - - run: - name: Build buck2 binary (release) - command: | - mkdir /tmp/artifacts - cargo build --bin=buck2 --release -Z unstable-options --out-dir=/tmp/artifacts - - run_test_py: - description: Run test.py - steps: - - run: - name: Run test.py - command: python3 test.py --ci --git --buck2=/tmp/artifacts/buck2 - - build_bootstrap: - description: Build `buck2` with `buck2` - steps: - - run: - name: Build `buck2` with `buck2` - command: | - /tmp/artifacts/buck2 build :buck2 -v 2 - - build_example_no_prelude: - description: Build example/no_prelude directory - steps: - - run: - name: Build example/no_prelude directory - command: | - cd examples/no_prelude - /tmp/artifacts/buck2 build //... -v 2 - - build_example_conan: - description: Buile examples/toolchains/conan_toolchain - steps: - - run: - name: Build examples/toolchains/conan_toolchain - command: | - cd examples/toolchains/conan_toolchain - /tmp/artifacts/buck2 init - cp -r ../../../prelude prelude - # Generate Conan imports. TODO[AH] Make that unnecessary. - PATH="/tmp/artifacts:$PATH" /tmp/artifacts/buck2 run //cpp/conan:update -v 2 - /tmp/artifacts/buck2 build //... -v 2 - /tmp/artifacts/buck2 test //... -v 2 - - build_example_zig: - description: Buile examples/toolchains/cxx_zig_toolchain - steps: - - run: - name: Build examples/toolchains/cxx_zig_toolchain - command: | - cd examples/toolchains/cxx_zig_toolchain - /tmp/artifacts/buck2 init - cp -r ../../../prelude prelude - /tmp/artifacts/buck2 build //... -v 2 - /tmp/artifacts/buck2 run //:main -v 2 - - -version: 2.1 -orbs: - win: circleci/windows@5.0 -jobs: - linux-build-and-test: - description: | - Build and test all with cargo for Linux - docker: - - image: cimg/rust:1.65.0 - resource_class: 2xlarge - steps: - - checkout - - setup_linux_env - - build_debug - - run_test_py - - linux-build-examples: - description: Build example projects - docker: - - image: cimg/rust:1.65.0 - resource_class: xlarge - steps: - - checkout - - setup_linux_env - - init_opam - - build_release - - run: - name: Build example/prelude directory - command: | - cd examples/with_prelude - /tmp/artifacts/buck2 init - cp -r ../../prelude prelude - # Additional setup for ocaml - source ./ocaml-setup.sh - /tmp/artifacts/buck2 build //... -v 2 - /tmp/artifacts/buck2 test //... -v 2 - - build_example_conan - - build_example_no_prelude - - setup_reindeer - - build_bootstrap - - macos-build-and-test: - description: | - Build all with cargo for macOS - macos: - xcode: "14.2.0" # macOS version 12.6 (see https://circleci.com/docs/using-macos/) - resource_class: macos.m1.medium.gen1 - steps: - - checkout - - setup_macos_env - - build_debug - - run_test_py - - macos-build-examples: - description: Build example projects - macos: - xcode: "14.2.0" - resource_class: macos.m1.medium.gen1 - steps: - - checkout - - setup_macos_env - - init_opam - - build_release - - run: - name: Build example/prelude directory - command: | - cd examples/with_prelude - /tmp/artifacts/buck2 init - cp -r ../../prelude prelude - # Additional setup for ocaml - source ./ocaml-setup.sh - /tmp/artifacts/buck2 build //... -v 2 - /tmp/artifacts/buck2 test //... -v 2 - - build_example_conan - - build_example_no_prelude - - setup_reindeer - - build_bootstrap - - windows-build-and-test: - description: | - Build and test all with cargo for Windows - executor: - name: win/default - size: "xlarge" - shell: powershell.exe - steps: - - checkout - - setup_windows_env - - build_debug - - run_test_py - - windows-build-examples: - description: Build example projects - executor: - name: win/default - size: "xlarge" - shell: powershell.exe - steps: - - checkout - - setup_windows_env - - build_release - - run: - name: Build example/prelude directory - command: | - cd examples/with_prelude - /tmp/artifacts/buck2 init - copy-item -Path $env:CIRCLE_WORKING_DIRECTORY\prelude -Destination prelude -Recurse - /tmp/artifacts/buck2 build //... -v 2 - /tmp/artifacts/buck2 test //... -v 2 - - build_example_no_prelude - - setup_reindeer - - build_bootstrap - -workflows: - build-and-test: - jobs: - - linux-build-and-test - - linux-build-examples - - macos-build-and-test - - macos-build-examples - - windows-build-and-test - - windows-build-examples diff --git a/.github/actions/build_bootstrap/action.yml b/.github/actions/build_bootstrap/action.yml new file mode 100644 index 0000000000000..ac8d7bda87622 --- /dev/null +++ b/.github/actions/build_bootstrap/action.yml @@ -0,0 +1,7 @@ +name: build_bootstrap +runs: + using: composite + steps: + - name: Build `buck2` with `buck2` + run: "$RUNNER_TEMP/artifacts/buck2 build :buck2 -v 2" + shell: bash diff --git a/.github/actions/build_debug/action.yml b/.github/actions/build_debug/action.yml new file mode 100644 index 0000000000000..aa4c7dc0309ea --- /dev/null +++ b/.github/actions/build_debug/action.yml @@ -0,0 +1,10 @@ +name: build_debug +description: Build buck2 binary (debug) +runs: + using: composite + steps: + - name: Build buck2 binary (debug) + run: |- + mkdir $RUNNER_TEMP/artifacts + cargo build --bin=buck2 -Z unstable-options --out-dir=$RUNNER_TEMP/artifacts + shell: bash diff --git a/.github/actions/build_example_conan/action.yml b/.github/actions/build_example_conan/action.yml new file mode 100644 index 0000000000000..6213ab4a5176f --- /dev/null +++ b/.github/actions/build_example_conan/action.yml @@ -0,0 +1,14 @@ +name: build_example_conan +runs: + using: composite + steps: + - name: Build examples/toolchains/conan_toolchain + run: |- + cd examples/toolchains/conan_toolchain + $RUNNER_TEMP/artifacts/buck2 init + cp -r ../../../prelude prelude + # Generate Conan imports. TODO[AH] Make that unnecessary. + PATH="$RUNNER_TEMP/artifacts:$PATH" $RUNNER_TEMP/artifacts/buck2 run //cpp/conan:update -v 2 + $RUNNER_TEMP/artifacts/buck2 build //... -v 2 + $RUNNER_TEMP/artifacts/buck2 test //... -v 2 + shell: bash diff --git a/.github/actions/build_example_no_prelude/action.yml b/.github/actions/build_example_no_prelude/action.yml new file mode 100644 index 0000000000000..7f3e09a5effa8 --- /dev/null +++ b/.github/actions/build_example_no_prelude/action.yml @@ -0,0 +1,9 @@ +name: build_example_no_prelude +runs: + using: composite + steps: + - name: Build example/no_prelude directory + run: |- + cd examples/no_prelude + $RUNNER_TEMP/artifacts/buck2 build //... -v 2 + shell: bash diff --git a/.github/actions/build_release/action.yml b/.github/actions/build_release/action.yml new file mode 100644 index 0000000000000..d332608a454e4 --- /dev/null +++ b/.github/actions/build_release/action.yml @@ -0,0 +1,10 @@ +name: build_release +description: Build buck2 binary (release) +runs: + using: composite + steps: + - name: Build buck2 binary (release) + run: |- + mkdir $RUNNER_TEMP/artifacts + cargo build --bin=buck2 --release -Z unstable-options --out-dir=$RUNNER_TEMP/artifacts + shell: bash diff --git a/.github/actions/init_opam/action.yml b/.github/actions/init_opam/action.yml new file mode 100644 index 0000000000000..30f3424627073 --- /dev/null +++ b/.github/actions/init_opam/action.yml @@ -0,0 +1,13 @@ +name: init_opam +description: Setup OPAM +runs: + using: composite + steps: + - name: Initialize OPAM + run: | + opam init --compiler=5.1.1 --disable-sandboxing -y + echo 'eval $(opam env)' >> ~/.bashrc + shell: bash + - name: Install OPAM packages + run: opam install menhir ppxlib -y + shell: bash diff --git a/.github/actions/print_versions/action.yml b/.github/actions/print_versions/action.yml new file mode 100644 index 0000000000000..df442f55cb416 --- /dev/null +++ b/.github/actions/print_versions/action.yml @@ -0,0 +1,12 @@ +name: print_versions +runs: + using: composite + steps: + - name: Version Info + run: |- + rustup show + rustc --version + cargo --version + rustup --version + python3 --version + shell: bash diff --git a/.github/actions/run_test_py/action.yml b/.github/actions/run_test_py/action.yml new file mode 100644 index 0000000000000..c1957417f59b0 --- /dev/null +++ b/.github/actions/run_test_py/action.yml @@ -0,0 +1,7 @@ +name: run_test_py +runs: + using: composite + steps: + - name: Run test.py + run: python3 test.py --ci --git --buck2=$RUNNER_TEMP/artifacts/buck2 + shell: bash diff --git a/.github/actions/setup_linux_env/action.yml b/.github/actions/setup_linux_env/action.yml new file mode 100644 index 0000000000000..e490aff68b091 --- /dev/null +++ b/.github/actions/setup_linux_env/action.yml @@ -0,0 +1,24 @@ +name: Setup Linux environment +description: Setup Linux environment +runs: + using: composite + steps: + - uses: SebRollen/toml-action@v1.0.2 + id: read_rust_toolchain + with: + file: rust-toolchain + field: toolchain.channel + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ steps.read_rust_toolchain.outputs.value }} + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: buck2-upload + - run: sudo apt-get update + shell: bash + - run: sudo apt-get install opam libzstd-dev python3-pip ghc + shell: bash + - name: Install conan + run: sudo pip3 install conan==1.* + shell: bash diff --git a/.github/actions/setup_macos_env/action.yml b/.github/actions/setup_macos_env/action.yml new file mode 100644 index 0000000000000..a0a9007b4b924 --- /dev/null +++ b/.github/actions/setup_macos_env/action.yml @@ -0,0 +1,15 @@ +name: setup_macos_env +description: Setup macOS environment +runs: + using: composite + steps: + - name: Install Rustup + run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=none + shell: bash + - name: Brew install + run: brew install cmake python3 coreutils opam llvm protobuf zstd ghc + shell: bash + - name: Install conan + run: sudo pip3 install conan==1.* + shell: bash + - uses: "./.github/actions/print_versions" diff --git a/.github/actions/setup_reindeer/action.yml b/.github/actions/setup_reindeer/action.yml new file mode 100644 index 0000000000000..ef9ecad4abfc7 --- /dev/null +++ b/.github/actions/setup_reindeer/action.yml @@ -0,0 +1,9 @@ +name: setup_reindeer +runs: + using: composite + steps: + - name: Install Reindeer + run: |- + cargo install --locked --git https://github.com/facebookincubator/reindeer reindeer + reindeer --third-party-dir shim/third-party/rust buckify + shell: bash diff --git a/.github/actions/setup_windows_env/action.yml b/.github/actions/setup_windows_env/action.yml new file mode 100644 index 0000000000000..874fe0c809ae2 --- /dev/null +++ b/.github/actions/setup_windows_env/action.yml @@ -0,0 +1,27 @@ +name: setup_windows_env +description: Setup Windows environment for building and testing +runs: + using: composite + steps: + - name: Install Rustup + run: |- + choco install -y rustup.install + write-output "[net]`ngit-fetch-with-cli = true" | out-file -append -encoding utf8 $Env:USERPROFILE/.cargo/config.toml + type $Env:USERPROFILE/.cargo/config.toml + shell: pwsh + - name: Create python3 symlink + run: New-Item -ItemType SymbolicLink -Path C:\ProgramData\chocolatey\bin\python3.exe -Target $(Get-Command python).Source + shell: pwsh + - name: Write Powershell profile + run: |- + $psProfileContent = @' + $vsPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -property installationPath + $llvmPath = Join-Path $vsPath "VC\Tools\Llvm\x64\bin" + $env:PATH = "$env:USERPROFILE\.cargo\bin;$llvmPath;" + $env:PATH + $env:TEMP = "$env:USERPROFILE\temp" + $env:TMP = $env:TEMP + '@ + Add-Content "$PsHome\profile.ps1" $psProfileContent + New-Item -ItemType Directory -Path "$env:USERPROFILE\temp" + shell: pwsh + - uses: "./.github/actions/print_versions" diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000000000..321c5da9f8456 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,93 @@ +name: Build and test +on: + push: + pull_request: +jobs: + linux-build-and-test: + runs-on: 4-core-ubuntu + steps: + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_linux_env + - uses: ./.github/actions/build_debug + - uses: ./.github/actions/run_test_py + macos-build-and-test: + runs-on: macos-latest + steps: + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.2.0 + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_macos_env + - uses: ./.github/actions/build_debug + - uses: ./.github/actions/run_test_py + windows-build-and-test: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_windows_env + - uses: ./.github/actions/build_debug + - uses: ./.github/actions/run_test_py + macos-build-examples: + runs-on: macos-latest + steps: + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.2.0 + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_macos_env + - uses: ./.github/actions/init_opam + - uses: ./.github/actions/build_release + - name: Build example/prelude directory + run: |- + eval $(opam env) + cd examples/with_prelude + $RUNNER_TEMP/artifacts/buck2 init + cp -r ../../prelude prelude + source ./ocaml-setup.sh + $RUNNER_TEMP/artifacts/buck2 build //... -v 2 + $RUNNER_TEMP/artifacts/buck2 test //... -v 2 + - uses: ./.github/actions/build_example_no_prelude + - uses: ./.github/actions/setup_reindeer + - uses: ./.github/actions/build_bootstrap + linux-build-examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_linux_env + - uses: ./.github/actions/init_opam + - uses: ./.github/actions/build_release + - name: Build example/prelude directory + run: |- + eval $(opam env) + cd examples/with_prelude + $RUNNER_TEMP/artifacts/buck2 init + cp -r ../../prelude prelude + source ./ocaml-setup.sh + $RUNNER_TEMP/artifacts/buck2 build //... -v 2 + $RUNNER_TEMP/artifacts/buck2 test //... -v 2 + - uses: ./.github/actions/build_example_conan + - uses: ./.github/actions/build_example_no_prelude + - uses: ./.github/actions/setup_reindeer + - uses: ./.github/actions/build_bootstrap + windows-build-examples: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: ./.github/actions/setup_windows_env + - uses: ./.github/actions/build_release + - name: Build example/prelude directory + run: |- + cd examples/with_prelude + & $Env:RUNNER_TEMP/artifacts/buck2 init + copy-item -Path $env:GITHUB_WORKSPACE\prelude -Destination prelude -Recurse + & $Env:RUNNER_TEMP/artifacts/buck2 build //... -v 2 + & $Env:RUNNER_TEMP/artifacts/buck2 test //... -v 2 + - uses: ./.github/actions/build_example_no_prelude + - name: Configure CARGO_HOME + run: |- + echo CARGO_HOME=$GITHUB_WORKSPACE/.cargo >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/.cargo/bin >> $GITHUB_PATH + shell: + bash + - uses: ./.github/actions/setup_reindeer + - uses: ./.github/actions/build_bootstrap diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9dd7dd72ced..72e48e52cbcab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,3 @@ # Buck2 -* Initial version. +- Initial version. diff --git a/Cargo.toml b/Cargo.toml index dcdd0352a9f95..cbe668fdab05c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,7 +115,7 @@ assert_matches = "1.5" async-compression = { version = "0.4.1", features = ["tokio", "gzip", "zstd"] } async-condvar-fair = { version = "1.0", features = ["parking_lot_0_11", "tokio"] } async-recursion = "1.0" -async-scoped = { version = "0.7.1", features = ["use-tokio"] } +async-scoped = { version = "0.8", features = ["use-tokio"] } async-trait = "0.1.24" atomic = "0.5.1" backtrace = "0.3.51" @@ -139,7 +139,7 @@ crossbeam-epoch = "0.9.7" crossterm = "0.27" csv = "1.1" ctor = "0.1.16" -dashmap = "4.0.2" +dashmap = "5.5.3" debugserver-types = "0.5.0" derivative = "2.2" derive_more = "0.99.3" @@ -159,9 +159,10 @@ fnv = "1.0.7" fs4 = { version = "0.6", features = ["sync"] } futures = { version = "0.3.28", features = ["async-await", "compat"] } futures-intrusive = "0.4" +fxhash = "0.2.1" glob = "0.3.0" globset = "0.4.10" -hashbrown = { version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.3", features = ["raw"] } hex = "0.4.3" higher-order-closure = "0.0.5" hostname = "0.3.1" @@ -212,6 +213,7 @@ once_cell = "1.8" os_str_bytes = { version = "6.6.0", features = ["conversions"] } parking_lot = { version = "0.11.2", features = ["send_guard"] } paste = "1.0" +pathdiff = "0.2" perf-event = "0.4" perf-event-open-sys = "4.0" pin-project = "0.4.29" @@ -232,17 +234,17 @@ ref-cast = "1.0.0" regex = "1.5.4" relative-path = { version = "1.7.0", features = ["serde"] } rusqlite = { version = "0.29.0", features = ["bundled"] } -rustls = "0.21.0" +rustls = "0.21.5" rustls-native-certs = { package = "rustls-native-certs", version = "0.6.2" } rustls-pemfile = { package = "rustls-pemfile", version = "1.0.0" } rustyline = "11.0" scopeguard = "1.0.0" sequence_trie = "0.3.6" -serde = { version = "1.0", features = ["derive"] } +serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0.48" sha1 = "0.10" sha2 = "0.10" -shlex = "1.0" +shlex = "1.3" siphasher = "0.3.3" slab = "0.4.7" slog = "2.7.0" @@ -276,7 +278,7 @@ tower-layer = "0.3.1" tower-service = "0.3.2" tracing = "0.1.22" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -triomphe = "0.1.8" +triomphe = "0.1.11" trybuild = "1.0.56" twox-hash = "1.6.1" unicode-segmentation = "1.7" @@ -309,10 +311,10 @@ lock_free_hashtable = { version = "0.1.0", path = "shed/lock_free_hashtable" } lock_free_vec = { path = "shed/lock_free_vec" } provider = { path = "shed/provider" } remote_execution = { path = "remote_execution/oss/re_grpc" } -starlark = { version = "0.10.0", path = "starlark-rust/starlark" } -starlark_lsp = { version = "0.10.0", path = "starlark-rust/starlark_lsp" } -starlark_map = { version = "0.10.0", path = "starlark-rust/starlark_map" } -starlark_syntax = { version = "0.10.0", path = "starlark-rust/starlark_syntax" } +starlark = { version = "0.12.0", path = "starlark-rust/starlark" } +starlark_lsp = { version = "0.12.0", path = "starlark-rust/starlark_lsp" } +starlark_map = { version = "0.12.0", path = "starlark-rust/starlark_map" } +starlark_syntax = { version = "0.12.0", path = "starlark-rust/starlark_syntax" } buck2_action_impl = { path = "app/buck2_action_impl" } buck2_action_metadata_proto = { path = "app/buck2_action_metadata_proto" } diff --git a/HACKING.md b/HACKING.md index 28f1c82015f35..fb16ccbe2a489 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,7 +1,8 @@ # Tips and tricks for hacking on Buck2 You might have been lead here by reading [CONTRIBUTING.md](/CONTRIBUTING.md). If -not, please read that as well! That will give you the high level overview; this document is all about the needed elbow grease you'll have to apply. +not, please read that as well! That will give you the high level overview; this +document is all about the needed elbow grease you'll have to apply. ## Building the code @@ -21,8 +22,8 @@ cargo install --path=app/buck2 Or, alternatively, install it directly from GitHub: ```sh -rustup install nightly-2023-10-01 -cargo +nightly-2023-10-01 install --git https://github.com/facebook/buck2.git buck2 +rustup install nightly-2023-11-10 +cargo +nightly-2023-11-10 install --git https://github.com/facebook/buck2.git buck2 ``` ### Side note: using [Nix] to compile the source @@ -110,8 +111,8 @@ have written. Some rules: ### Error messages - Names (of variables, targets, files, etc) should be quoted with backticks, - e.g. ``Variable `x` not defined``. -- Lists should use square brackets, e.g. ``Available targets: [`aa`, `bb`]``. + e.g. `` Variable `x` not defined ``. +- Lists should use square brackets, e.g. `` Available targets: [`aa`, `bb`] ``. - Error messages should start with an upper case letter. Error messages should not end with a period. @@ -120,10 +121,10 @@ have written. Some rules: Most code is shared as-is between open source and the internal Meta version of Buck2. However, there are some exceptions: -* The open-source remote execution client is different, because our internal - one works with custom servers/infrastructure that is not publicly available. -* There are places controlled with `is_open_source()` which change configuration +- The open-source remote execution client is different, because our internal one + works with custom servers/infrastructure that is not publicly available. +- There are places controlled with `is_open_source()` which change configuration between the internal and open source versions. -* Some places use `@oss-enable` or `@oss-disable` to comment/uncomment lines - of code. The internal code is visible, but the comment markers are moved - during export/import of code. +- Some places use `@oss-enable` or `@oss-disable` to comment/uncomment lines of + code. The internal code is visible, but the comment markers are moved during + export/import of code. diff --git a/README.md b/README.md index dedf7eab75b63..5c53ac9756185 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,15 @@ # Buck2: fast multi-language build system -![Version] -![License] -[![Build Status]][CI] +![Version] ![License] [![Build Status]][CI] -[Version]: https://img.shields.io/badge/release-unstable,%20"Developer%20Edition"-orange.svg -[License]: https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blueviolet.svg -[Build Status]: https://img.shields.io/circleci/build/github/facebook/buck2 -[CI]: https://app.circleci.com/pipelines/github/facebook/buck2 +[Version]: + https://img.shields.io/badge/release-unstable,%20"Developer%20Edition"-orange.svg +[License]: + https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blueviolet.svg +[Build Status]: + https://github.com/facebook/buck2/actions/workflows/build-and-test.yml/badge.svg +[CI]: https://github.com/facebook/buck2/actions/workflows/build-and-test.yml Homepage  •  Getting Started  •  Contributing @@ -31,13 +32,13 @@ already exist? complete, or 0.1 seconds: when you have to build things, Buck2 doesn't waste time — it calculates the critical path and gets out of the way, with minimal overhead. It's not just the core design, but also careful attention to - detail that makes Buck2 so snappy. Buck2 is up to 2x faster than Buck1 *in - practice*[^perf-note]. So you spend more time iterating, and less time + detail that makes Buck2 so snappy. Buck2 is up to 2x faster than Buck1 _in + practice_[^perf-note]. So you spend more time iterating, and less time waiting. - **Hermetic**. When using Remote Execution[^hermetic-re-only], Buck2 becomes - *hermetic*: it is required for a build rule to correctly declare all of its - inputs; if they aren't specified correctly (e.g. a `.c` file neeads a `.h` - file that isn't correctly specified), the build will fail. This enforced + _hermetic_: it is required for a build rule to correctly declare all of its + inputs; if they aren't specified correctly (e.g. a `.c` file needs a `.h` file + that isn't correctly specified), the build will fail. This enforced correctness helps avoids entire classes of errors that most build systems allow, and helps ensure builds work everywhere for all users. And Buck2 correctly tracks dependencies with far better accuracy than Buck1, in more @@ -49,22 +50,24 @@ already exist? But then how do you run test suites, code coverage, or query code databases? Buck2 is designed to support multiple languages from the start, with abstractions for interoperation. And because it's completely scriptable, and - *users* can implement language support — it's incredibly flexible. Now + _users_ can implement language support — it's incredibly flexible. Now your Python library can depend on an OCaml library, and your OCaml library can depend on a Rust crate — and with a single build tool, you have a consistent UX to build and test and integrate all of these components. -[^perf-note]: This number comes from internal usage of Buck1 versus Buck2 at - Meta. Please note that *appropriate* comparisons with systems like Bazel - have yet to be performed; Buck1 is the baseline because it's simply what - existed and what had to be replaced. Please benchmark Buck2 against your - favorite tools and let us know how it goes! +[^perf-note]: + This number comes from internal usage of Buck1 versus Buck2 at Meta. Please + note that _appropriate_ comparisons with systems like Bazel have yet to be + performed; Buck1 is the baseline because it's simply what existed and what + had to be replaced. Please benchmark Buck2 against your favorite tools and + let us know how it goes! -[^hermetic-re-only]: Buck2 currently does not sandbox *local-only* build steps; - in contrast, Buck2 using Remote Execution is *always* hermetic by design. - The vast majority of build rules are remote compatible, as well. Despite - that, we hope to lift this restriction in the (hopefully short-term) future - so that local-only builds are hermetic as well. +[^hermetic-re-only]: + Buck2 currently does not sandbox _local-only_ build steps; in contrast, + Buck2 using Remote Execution is _always_ hermetic by design. The vast + majority of build rules are remote compatible, as well. Despite that, we + hope to lift this restriction in the (hopefully short-term) future so that + local-only builds are hermetic as well. If you're familiar with systems like Buck1, [Bazel](https://bazel.build/), or [Pants](https://www.pantsbuild.org/) — then Buck2 will feel warm and cozy, @@ -90,8 +93,8 @@ rest of the pack, including: build systems and incremental computation. - And more! -If these headline features make you interested — check out the [Getting -Started](https://buck2.build/docs/getting_started/) guide! +If these headline features make you interested — check out the +[Getting Started](https://buck2.build/docs/getting_started/) guide! ## 🚧🚧🚧 **Warning** 🚧🚧🚧 — rough terrain lies ahead @@ -115,10 +118,10 @@ Please provide feedback by submitting [issues and questions!](/issues) ## Installing Buck2 -You can get started by downloading the [latest buck2 -binary](https://github.com/facebook/buck2/releases/tag/latest) for your -platform. The `latest` tag always refers to a recent commit; it is updated on -every single push to the GitHub repository, so it will always be a recent +You can get started by downloading the +[latest buck2 binary](https://github.com/facebook/buck2/releases/tag/latest) for +your platform. The `latest` tag always refers to a recent commit; it is updated +on every single push to the GitHub repository, so it will always be a recent version. You can also compile Buck2 from source, if a binary isn't immediately available @@ -126,8 +129,8 @@ for your use; check out the [HACKING.md](./HACKING.md) file for information. ## Terminology conventions -Frequently used terms and their definitions can be found on the [glossary -page](https://buck2.build/docs/concepts/glossary/). +Frequently used terms and their definitions can be found on the +[glossary page](https://buck2.build/docs/concepts/glossary/). ## License diff --git a/action_error_handler/java/java_error_handler.bzl b/action_error_handler/java/java_error_handler.bzl new file mode 100644 index 0000000000000..bb91c595082b5 --- /dev/null +++ b/action_error_handler/java/java_error_handler.bzl @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@fbsource//tools/build_defs/android/action_error_handler:android_di_error_handler.bzl", "android_di_error_handler") + +def java_error_handler(ctx: ActionErrorCtx) -> list[ActionSubError]: + categories = [] + + categories += android_di_error_handler(ctx) + + return categories diff --git a/action_error_handler/kotlin/kotlin_error_handler.bzl b/action_error_handler/kotlin/kotlin_error_handler.bzl new file mode 100644 index 0000000000000..da0bcef797d2c --- /dev/null +++ b/action_error_handler/kotlin/kotlin_error_handler.bzl @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@fbsource//tools/build_defs/android/action_error_handler:android_di_error_handler.bzl", "android_di_error_handler") + +def kotlin_error_handler(ctx: ActionErrorCtx) -> list[ActionSubError]: + categories = [] + + categories += android_di_error_handler(ctx) + + return categories diff --git a/allocative/README.md b/allocative/README.md index 56fe805b18506..2c13bd7b2180e 100644 --- a/allocative/README.md +++ b/allocative/README.md @@ -1,41 +1,42 @@ # Allocative: memory profiler for Rust -This crate implements a lightweight memory profiler which allows -object traversal and memory size introspection. +This crate implements a lightweight memory profiler which allows object +traversal and memory size introspection. ## Usage `Allocative` trait (typically implemented with proc-macro) is introspectable: -`Allocative` values can be traversed and their size and sizes of referenced objects -can be collected. +`Allocative` values can be traversed and their size and sizes of referenced +objects can be collected. -This crate provides a few utilities to work with such objects, -the main of such utilities is flame graph builder which produces flame graph -(see the crate documentation) like this: +This crate provides a few utilities to work with such objects, the main of such +utilities is flame graph builder which produces flame graph (see the crate +documentation) like this: ![sample-flamegraph.png](sample-flamegraph.png) ## How it is different from other call-stack malloc profilers like jemalloc heap profiler -Allocative is not a substitute for call stack malloc profiler, -it provides a different view of memory usage. +Allocative is not a substitute for call stack malloc profiler, it provides a +different view of memory usage. Here are some differences between allocative and call-stack malloc profiler: -* Allocative requires implementation of `Allocative` trait for each type - which needs to be measured, and some setup in the program to enable it is needed -* Allocative flamegraph shows object by object tree, not by call stack -* Allocative shows gaps in allocated memory, - e.g. spare capacity of collections or too large padding in structs or enums -* Allocative allows profiling of non-malloc allocations - (for example, allocations within [bumpalo](https://github.com/fitzgen/bumpalo) bumps) -* Allocative allows profiling of memory for subset of the process data - (for example, measure the size of RPC response before serialization) +- Allocative requires implementation of `Allocative` trait for each type which + needs to be measured, and some setup in the program to enable it is needed +- Allocative flamegraph shows object by object tree, not by call stack +- Allocative shows gaps in allocated memory, e.g. spare capacity of collections + or too large padding in structs or enums +- Allocative allows profiling of non-malloc allocations (for example, + allocations within [bumpalo](https://github.com/fitzgen/bumpalo) bumps) +- Allocative allows profiling of memory for subset of the process data (for + example, measure the size of RPC response before serialization) ## Runtime overhead -When allocative is used, binary size is slightly increased due to implementations -of [`Allocative`] trait, but it has no runtime/memory overhead when it is enabled but not used. +When allocative is used, binary size is slightly increased due to +implementations of [`Allocative`] trait, but it has no runtime/memory overhead +when it is enabled but not used. ## Source code @@ -45,5 +46,5 @@ is synchronized to GitHub. The main copy is ## License -Allocative is both MIT and Apache License, Version 2.0 licensed, -as found in the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. +Allocative is both MIT and Apache License, Version 2.0 licensed, as found in the +[LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. diff --git a/allocative/allocative/Cargo.toml b/allocative/allocative/Cargo.toml index a733c098ec303..84947be11b920 100644 --- a/allocative/allocative/Cargo.toml +++ b/allocative/allocative/Cargo.toml @@ -6,20 +6,20 @@ edition = "2021" license = { workspace = true } name = "allocative" repository = "https://github.com/facebookexperimental/allocative" -version = "0.3.1" +version = "0.3.2" [dependencies] -allocative_derive = { path = "../allocative_derive", version = "=0.3.1" } +allocative_derive = { path = "../allocative_derive", version = "=0.3.2" } ctor = { workspace = true } anyhow = { version = "1.0.65", optional = true } bumpalo = { version = "3.11.1", optional = true } compact_str = { version = "0.6.1", optional = true } -dashmap = { version = "4.0.2", optional = true } +dashmap = { version = "5.5.3", optional = true } either = { version = "1.8", optional = true } futures = { version = "0.3.24", optional = true } -hashbrown = { version = "0.12.3", optional = true } +hashbrown = { version = "0.14.3", optional = true } indexmap = { version = "1.9.1", optional = true } num-bigint = { version = "0.4.3", optional = true } once_cell = { version = "1.15.0", optional = true } diff --git a/allocative/allocative/src/allocative_trait.rs b/allocative/allocative/src/allocative_trait.rs index 123c5d3d902de..fe83aad436178 100644 --- a/allocative/allocative/src/allocative_trait.rs +++ b/allocative/allocative/src/allocative_trait.rs @@ -25,13 +25,14 @@ use crate::Visitor; /// } /// ``` /// -/// Proc macro supports two attributes: `#[allocative(skip)]` and `#[allocative(bound = "")]`. +/// Proc macro supports two attributes: `#[allocative(skip)]` and +/// `#[allocative(bound = "...")]`. /// /// ## `#[allocative(skip)]` /// -/// `#[allocative(skip)]` can be used to skip field from traversal -/// (for example, to skip fields which are not `Allocative`, -/// and can be skipped because they are cheap). +/// `#[allocative(skip)]` can be used to skip field from traversal (for example, +/// to skip fields which are not `Allocative`, and can be skipped because they +/// are cheap). /// /// ``` /// use allocative::Allocative; @@ -46,13 +47,19 @@ use crate::Visitor; /// } /// ``` /// -/// ## `#[allocative(bound = "")]` +/// ## `#[allocative(bound = "...")]` +/// +/// `#[allocative(bound = "...")]` can be used to overwrite the bounds that are +/// added to the generics of the implementation. +/// +/// An empty string (`#[allocative(bound = "")]`) simply erases all bounds. It +/// adds all type variables found in the type to the list of generics but with +/// an empty bound. As an example /// -/// `#[allocative(bound = "")]` can be used to not add `T: Allocative` bound -/// to `Allocative` trait implementation, like this: /// /// ``` /// use std::marker::PhantomData; +/// /// use allocative::Allocative; /// /// struct Unsupported; @@ -62,9 +69,71 @@ use crate::Visitor; /// struct Baz { /// _marker: PhantomData, /// } +/// ``` +/// +/// Would generate an instance +/// +/// ```ignore +/// impl Allocative for Baz { ... } +/// ``` +/// +/// Alternatively you can use the string to provide custom bounds. The string in +/// this case is used *verbatim* as the bounds, which affords great flexibility, +/// but also necessitates that all type variables must be mentioned or will be +/// unbound (compile error). As an example we may derive a size of a `HashMap` +/// by ignoring the hasher type. +/// +/// +/// ```ignore +/// #[allocative(bound = "K: Allocative, V:Allocative, S")] +/// struct HashMap { +/// ... +/// } +/// ``` +/// +/// Which generates +/// +/// ```ignore +/// impl Allocative for HashMap { +/// ... +/// } +/// ``` +/// +/// ## `#[allocative(visit = ...)]` +/// +/// This annotation is used to provide a custom visit method for a given field. This +/// is especially useful if the type of the field does not implement `Allocative`. +/// +/// The annotation takes the path to a method with a signature `for<'a, 'b>(&T, &'a +/// mut allocative::Visitor<'b>)` where `T` is the type of the field. The function +/// you provide is basically the same as if you implemented [`Allocative::visit`]. +/// +/// As an example +/// +/// ``` +/// use allocative::Allocative; +/// use allocative::Key; +/// use allocative::Visitor; +/// // use third_party_lib::Unsupported; +/// # struct Unsupported(T); +/// # impl Unsupported { +/// # fn iter_elems(&self) -> &[T] { &[] } +/// # } +/// +/// #[derive(Allocative)] +/// struct Bar { +/// #[allocative(visit = visit_unsupported)] +/// unsupported: Unsupported, +/// } /// -/// // So `Baz` is `Allocative` even though `Unsupported` is not. -/// let allocative: &dyn Allocative = &Baz:: { _marker: PhantomData }; +/// fn visit_unsupported<'a, 'b>(u: &Unsupported, visitor: &'a mut Visitor<'b>) { +/// const ELEM_KEY: Key = Key::new("elements"); +/// let mut visitor = visitor.enter_self(u); +/// for element in u.iter_elems() { +/// visitor.visit_field(ELEM_KEY, element); +/// } +/// visitor.exit() +/// } /// ``` pub trait Allocative { fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>); diff --git a/allocative/allocative/src/flamegraph.rs b/allocative/allocative/src/flamegraph.rs index 67a63c9f4f15e..30cb7006d775e 100644 --- a/allocative/allocative/src/flamegraph.rs +++ b/allocative/allocative/src/flamegraph.rs @@ -23,6 +23,8 @@ use crate::visitor::VisitorImpl; use crate::Allocative; /// Node in flamegraph tree. +/// +/// Can be written to flamegraph format with [`write`](FlameGraph::write). #[derive(Debug, Default, Clone)] pub struct FlameGraph { children: HashMap, @@ -82,7 +84,10 @@ impl FlameGraph { } } - /// Write flamegraph in format suitable for `flamegraph.pl` or `inferno`. + /// Write flamegraph in format suitable for [`flamegraph.pl`] or [inferno]. + /// + /// [flamegraph.pl]: https://github.com/brendangregg/FlameGraph + /// [inferno]: https://github.com/jonhoo/inferno pub fn write(&self) -> String { let mut r = String::new(); self.write_flame_graph_impl(&[], &mut r); @@ -276,18 +281,22 @@ unsafe impl Send for VisitedSharedPointer {} /// # Example /// /// ``` -/// use allocative::FlameGraphBuilder; /// use allocative::Allocative; +/// use allocative::FlameGraphBuilder; /// /// #[derive(Allocative)] /// struct Foo { /// data: String, /// }; /// -/// let foo1 = Foo { data: "Hello, world!".to_owned() }; -/// let foo2 = Foo { data: "Another message!".to_owned() }; +/// let foo1 = Foo { +/// data: "Hello, world!".to_owned(), +/// }; +/// let foo2 = Foo { +/// data: "Another message!".to_owned(), +/// }; /// -/// let mut flamegraph = FlameGraphBuilder::default(); +/// let mut flamegraph = FlameGraphBuilder::default(); /// flamegraph.visit_root(&foo1); /// flamegraph.visit_root(&foo2); /// let flamegraph_src = flamegraph.finish().flamegraph(); diff --git a/allocative/allocative/src/golden.rs b/allocative/allocative/src/golden.rs index bee5ab960bd55..6afcde98ec77c 100644 --- a/allocative/allocative/src/golden.rs +++ b/allocative/allocative/src/golden.rs @@ -10,6 +10,7 @@ #![cfg(test)] use std::env; +use std::fmt::Write; use std::fs; use crate::Allocative; @@ -50,8 +51,10 @@ fn make_golden(value: &T) -> (String, String) { "{header}{flamegraph}", header = golden_header() .lines() - .map(|line| format!("# {}\n", line)) - .collect::() + .fold(String::new(), |mut output, line| { + let _ = writeln!(output, "# {}", line); + output + }) ); let flamegraph_svg = flamegraph_svg.replace( diff --git a/allocative/allocative/src/impls/smallvec.rs b/allocative/allocative/src/impls/smallvec.rs index 2bb13eaf96e3f..8c1f1ab55cd67 100644 --- a/allocative/allocative/src/impls/smallvec.rs +++ b/allocative/allocative/src/impls/smallvec.rs @@ -19,7 +19,7 @@ use crate::visitor::Visitor; impl Allocative for SmallVec where - A: Array + 'static, + A: Array, A::Item: Allocative, { fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) { diff --git a/allocative/allocative/src/impls/std/any.rs b/allocative/allocative/src/impls/std/any.rs new file mode 100644 index 0000000000000..1e67f46ee08bf --- /dev/null +++ b/allocative/allocative/src/impls/std/any.rs @@ -0,0 +1,17 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use crate::allocative_trait::Allocative; +use crate::visitor::Visitor; + +impl Allocative for std::any::TypeId { + fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) { + visitor.visit_simple_sized::(); + } +} diff --git a/allocative/allocative/src/impls/std/mod.rs b/allocative/allocative/src/impls/std/mod.rs index 65f5b4edcbacb..49e07bb628c18 100644 --- a/allocative/allocative/src/impls/std/mod.rs +++ b/allocative/allocative/src/impls/std/mod.rs @@ -7,6 +7,7 @@ * of this source tree. */ +mod any; mod cell; mod collections; mod function; diff --git a/allocative/allocative/src/impls/std/sync.rs b/allocative/allocative/src/impls/std/sync.rs index 529687ed9b7fa..0bcfa673ec9b1 100644 --- a/allocative/allocative/src/impls/std/sync.rs +++ b/allocative/allocative/src/impls/std/sync.rs @@ -52,7 +52,8 @@ impl Allocative for Arc { Arc::as_ptr(self) as *const (), ); if let Some(mut visitor) = visitor { - struct ArcInner(AtomicUsize, AtomicUsize, ()); + #[allow(dead_code)] // Only used for its size + struct ArcInner(AtomicUsize, AtomicUsize); { let val: &T = self; let mut visitor = visitor.enter( diff --git a/allocative/allocative/src/lib.rs b/allocative/allocative/src/lib.rs index 388b8829ec14e..3fa8e7a4f1e45 100644 --- a/allocative/allocative/src/lib.rs +++ b/allocative/allocative/src/lib.rs @@ -73,3 +73,37 @@ pub use crate::visitor::Visitor; pub mod __macro_refs { pub use ctor; } + +/// Create a `const` of type `Key` with the provided `ident` as the value and +/// return that value. This allows the keys to be placed conveniently inline +/// without any performance hit because unlike calling `Key::new` this is +/// guaranteed to be evaluated at compile time. +/// +/// The main use case is manual implementations of [`Allocative`], like so: +/// +/// ``` +/// use allocative::ident_key; +/// use allocative::Allocative; +/// use allocative::Visitor; +/// +/// struct MyStruct { +/// foo: usize, +/// bar: Vec<()>, +/// } +/// +/// impl Allocative for MyStruct { +/// fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) { +/// let mut visitor = visitor.enter_self(self); +/// visitor.visit_field(ident_key!(foo), &self.foo); +/// visitor.visit_field(ident_key!(bar), &self.bar); +/// visitor.exit(); +/// } +/// } +/// ``` +#[macro_export] +macro_rules! ident_key { + ($name:ident) => {{ + const KEY: $crate::Key = $crate::Key::new(stringify!(name)); + KEY + }}; +} diff --git a/allocative/allocative/src/size_of.rs b/allocative/allocative/src/size_of.rs index 1662e07305adf..b31110d946629 100644 --- a/allocative/allocative/src/size_of.rs +++ b/allocative/allocative/src/size_of.rs @@ -29,7 +29,12 @@ use crate::Visitor; /// data: Vec, /// } /// -/// assert_eq!(3, allocative::size_of_unique_allocated_data(&Foo { data: vec![10, 20, 30] })); +/// assert_eq!( +/// 3, +/// allocative::size_of_unique_allocated_data(&Foo { +/// data: vec![10, 20, 30] +/// }) +/// ); /// ``` pub fn size_of_unique_allocated_data(root: &dyn Allocative) -> usize { struct SizeOfUniqueAllocatedDataVisitor { @@ -91,7 +96,12 @@ pub fn size_of_unique_allocated_data(root: &dyn Allocative) -> usize { /// data: Vec, /// } /// -/// assert_eq!(3 + std::mem::size_of::>(), allocative::size_of_unique(&Foo { data: vec![10, 20, 30] })); +/// assert_eq!( +/// 3 + std::mem::size_of::>(), +/// allocative::size_of_unique(&Foo { +/// data: vec![10, 20, 30] +/// }) +/// ); /// ``` pub fn size_of_unique(root: &T) -> usize where diff --git a/allocative/allocative/src/test_derive/bounds.rs b/allocative/allocative/src/test_derive/bounds.rs new file mode 100644 index 0000000000000..011cd533f8113 --- /dev/null +++ b/allocative/allocative/src/test_derive/bounds.rs @@ -0,0 +1,34 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use crate as allocative; +use crate::Allocative; + +#[derive(Allocative)] +#[allocative(bound = "K: Allocative, V:Allocative, S")] +struct HashMap { + map: std::collections::HashMap, +} + +#[derive(Allocative)] +#[allocative(bound = "S: Sized")] +struct CanBeUnsized { + #[allocative(visit = via_sized)] + s: Box, +} + +#[allow(clippy::borrowed_box)] +fn via_sized(s: &Box, visitor: &mut allocative::Visitor) { + visitor + .enter( + allocative::Key::new("s"), + std::mem::size_of_val(Box::as_ref(s)), + ) + .exit() +} diff --git a/allocative/allocative/src/test_derive/mod.rs b/allocative/allocative/src/test_derive/mod.rs index d3db100c88b87..6dff32f758f1a 100644 --- a/allocative/allocative/src/test_derive/mod.rs +++ b/allocative/allocative/src/test_derive/mod.rs @@ -10,6 +10,7 @@ #![cfg(test)] #![allow(dead_code)] +mod bounds; mod dst; mod skip; mod visit; diff --git a/allocative/allocative/src/visitor.rs b/allocative/allocative/src/visitor.rs index 5ec7fd6c39a6f..9fd302e5d5f01 100644 --- a/allocative/allocative/src/visitor.rs +++ b/allocative/allocative/src/visitor.rs @@ -154,8 +154,24 @@ impl<'a> Visitor<'a> { where 'a: 'b, { - let mut visitor = self.enter(name, mem::size_of_val::(field)); - field.visit(&mut visitor); + self.visit_field_with(name, mem::size_of_val::(field), |visitor| { + field.visit(visitor); + }) + } + + /// Similar to `visit_field` but instead of calling [`Allocative::visit`] for + /// whichever is the field type, you can provide a custom closure to call + /// instead. + /// + /// Useful if the field type does not implement [`Allocative`]. + pub fn visit_field_with<'b, 'f, F: for<'c, 'd> FnOnce(&'d mut Visitor<'c>)>( + &'b mut self, + name: Key, + field_size: usize, + visit: F, + ) { + let mut visitor = self.enter(name, field_size); + visit(&mut visitor); visitor.exit(); } @@ -190,25 +206,25 @@ impl<'a> Visitor<'a> { 'a: 'b, T: Allocative, { - let mut visitor = self.enter(CAPACITY_NAME, mem::size_of::() * capacity); - visitor.visit_slice(data); - visitor.visit_simple( - UNUSED_CAPACITY_NAME, - mem::size_of::() * capacity.wrapping_sub(data.len()), - ); - visitor.exit(); + self.visit_field_with(CAPACITY_NAME, mem::size_of::() * capacity, |visitor| { + visitor.visit_slice(data); + visitor.visit_simple( + UNUSED_CAPACITY_NAME, + mem::size_of::() * capacity.wrapping_sub(data.len()), + ); + }) } pub fn visit_generic_map_fields<'b, 'x, K: Allocative + 'x, V: Allocative + 'x>( &'b mut self, entries: impl IntoIterator, ) { - let mut visitor = self.enter_unique(DATA_NAME, mem::size_of::<*const ()>()); - for (k, v) in entries { - visitor.visit_field(KEY_NAME, k); - visitor.visit_field(VALUE_NAME, v); - } - visitor.exit(); + self.visit_field_with(DATA_NAME, mem::size_of::<*const ()>(), move |visitor| { + for (k, v) in entries { + visitor.visit_field(KEY_NAME, k); + visitor.visit_field(VALUE_NAME, v); + } + }) } pub fn visit_generic_set_fields<'b, 'x, K: Allocative + 'x>( @@ -217,11 +233,11 @@ impl<'a> Visitor<'a> { ) where 'a: 'b, { - let mut visitor = self.enter_unique(DATA_NAME, mem::size_of::<*const ()>()); - for k in entries { - visitor.visit_field(KEY_NAME, k); - } - visitor.exit(); + self.visit_field_with(DATA_NAME, mem::size_of::<*const ()>(), |visitor| { + for k in entries { + visitor.visit_field(KEY_NAME, k); + } + }) } fn exit_impl(&mut self) { diff --git a/allocative/allocative_derive/Cargo.toml b/allocative/allocative_derive/Cargo.toml index ce7aa72696db0..0dcf98809713a 100644 --- a/allocative/allocative_derive/Cargo.toml +++ b/allocative/allocative_derive/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" license = { workspace = true } name = "allocative_derive" repository = "https://github.com/facebookexperimental/allocative" -version = "0.3.1" +version = "0.3.2" [lib] proc-macro = true diff --git a/allocative/allocative_derive/src/derive_allocative.rs b/allocative/allocative_derive/src/derive_allocative.rs index 83493540b9421..6044c357dc138 100644 --- a/allocative/allocative_derive/src/derive_allocative.rs +++ b/allocative/allocative_derive/src/derive_allocative.rs @@ -7,8 +7,8 @@ * of this source tree. */ -use proc_macro::TokenStream; use proc_macro2::Ident; +use proc_macro2::Span; use quote::quote_spanned; use quote::ToTokens; use syn::parse::ParseStream; @@ -41,7 +41,7 @@ const fn hash(s: &str) -> u64 { hash } -pub(crate) fn derive_allocative(input: TokenStream) -> TokenStream { +pub(crate) fn derive_allocative(input: proc_macro::TokenStream) -> proc_macro::TokenStream { match derive_allocative_impl(input.into()) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), @@ -54,10 +54,9 @@ fn impl_generics( ) -> syn::Result { if let Some(bound) = &attrs.bound { if !bound.is_empty() { - return Err(syn::Error::new( - attrs.bound.span(), - "non-empty bound is not implemented", - )); + let span = attrs.span.unwrap_or_else(Span::call_site); + let bound = bound.parse::()?; + return Ok(quote_spanned! { span => < #bound > }); } } @@ -301,6 +300,7 @@ fn gen_visit_field( #[derive(Default)] struct AllocativeAttrs { + span: Option, skip: bool, bound: Option, visit: Option, @@ -319,6 +319,8 @@ fn extract_attrs(attrs: &[Attribute]) -> syn::Result { continue; } + opts.span = Some(attr.span()); + attr.parse_args_with(|input: ParseStream| { loop { if input.parse::().is_ok() { diff --git a/app/buck2/daemon_lifecycle.md b/app/buck2/daemon_lifecycle.md index 201a5296744d9..23f2bb529bd98 100644 --- a/app/buck2/daemon_lifecycle.md +++ b/app/buck2/daemon_lifecycle.md @@ -3,12 +3,12 @@ Buck runs a persistent daemon process (buckd) to reuse work between commands. Most work is done by the daemon process. When executing a buck command, the process running the command is a client to the buckd server. The buckd server -exposes a simple grpc service that the client uses to implement the various -buck commands. +exposes a simple grpc service that the client uses to implement the various buck +commands. -There's a small set of commands/arguments that don't require the daemon (`buck -help`, cli arg parse failures, `buck version`, ...), but most commands will -require it. +There's a small set of commands/arguments that don't require the daemon +(`buck help`, cli arg parse failures, `buck version`, ...), but most commands +will require it. For almost all commands, buck requires that the client and server are the same version of buck and may restart buckd to ensure that's the case. @@ -21,11 +21,11 @@ The daemon process has a simple startup. It will first daemonize itself and write its pid to a locked file "buckd.pid" in the "daemon directory" (a directory in `$HOME/.buck` specific to that repository+output directory). The file is locked exclusively by the daemon process until it exits. This means that -only a single daemon is allowed at a time. It redirects its stdout and stderr -to files in the daemon directory. +only a single daemon is allowed at a time. It redirects its stdout and stderr to +files in the daemon directory. -The daemon then starts up the grpc DaemonApi server. Once that is running, it will -write the port it is running on (along with some other information) to the +The daemon then starts up the grpc DaemonApi server. Once that is running, it +will write the port it is running on (along with some other information) to the "buckd.info" file in the daemon dir. Once that is done, the server is ready to be used. @@ -46,8 +46,8 @@ buckd server it will follow this approach: 3. send a `status()` request to get the version If there is an error during 1-3, or if there is a version mismatch the client -needs to (re)start the buck daemon. Otherwise, the client can continue as it -now has made a connection with a correctly versioned buckd. +needs to (re)start the buck daemon. Otherwise, the client can continue as it now +has made a connection with a correctly versioned buckd. When the client is killing or starting the buckd process, it will grab an exclusive lock on the "lifecycle.lock" file in the daemon directory to ensure @@ -62,19 +62,21 @@ To start/restart the buckd process, the client does: 5. wait for the daemon to start up and the grpc server to be ready 6. release the "lifecycle.lock" file -After that, it will repeat the connection steps (including verifying the -version after connecting). +After that, it will repeat the connection steps (including verifying the version +after connecting). # buck kill and other daemon restarts -If there are other invocations currently using the buck daemon when it is killed or -restarted by a client, those invocations will fail due to the early disconnection. +If there are other invocations currently using the buck daemon when it is killed +or restarted by a client, those invocations will fail due to the early +disconnection. -Generally, we support concurrent buck invocations using the same buck version, but -if there are concurrent invocations with different versions, they may unexpectedly -fail or otherwise work incorrectly. This is sufficient for the normal buck workflow -where the buckversion is checked into the repo, in that case, it's not expected that -buck commands will work across a rebase or other operation that changes the buckversion. +Generally, we support concurrent buck invocations using the same buck version, +but if there are concurrent invocations with different versions, they may +unexpectedly fail or otherwise work incorrectly. This is sufficient for the +normal buck workflow where the buckversion is checked into the repo, in that +case, it's not expected that buck commands will work across a rebase or other +operation that changes the buckversion. # correctness @@ -82,11 +84,13 @@ We have a couple of guarantees here. 1. Only a single buckd is running at a time 2. Only a single client is killing/starting a buckd at a time -3. A client only uses a buckd connection after making sure it has a compatible version - -The main way that we could run into issues would be if there are multiple clients -that are racing and they want different versions of buck. In that case, one -might cause the other two fail to connect to a buckd with the correct version -or one of the client's connections may be prematurely disconnected. A client **will not** -use a server with a mismatched version. While this is a failure, no expected workflow -would hit this case, all concurrent commands should be using the same buck version. +3. A client only uses a buckd connection after making sure it has a compatible + version + +The main way that we could run into issues would be if there are multiple +clients that are racing and they want different versions of buck. In that case, +one might cause the other two fail to connect to a buckd with the correct +version or one of the client's connections may be prematurely disconnected. A +client **will not** use a server with a mismatched version. While this is a +failure, no expected workflow would hit this case, all concurrent commands +should be using the same buck version. diff --git a/app/buck2/src/commands/daemon.rs b/app/buck2/src/commands/daemon.rs index 5033622d10b82..68649a0e9f777 100644 --- a/app/buck2/src/commands/daemon.rs +++ b/app/buck2/src/commands/daemon.rs @@ -11,7 +11,6 @@ use std::fs::File; use std::path::PathBuf; use std::process; use std::sync::Arc; -use std::thread; use std::time::Duration; use allocative::Allocative; @@ -41,6 +40,8 @@ use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; use buck2_starlark::server::server_starlark_command; +use buck2_util::threads::thread_spawn; +use buck2_util::tokio_runtime::new_tokio_runtime; use futures::channel::mpsc; use futures::channel::mpsc::UnboundedSender; use futures::pin_mut; @@ -305,9 +306,8 @@ impl DaemonCommand { } } - let mut builder = Builder::new_multi_thread(); + let mut builder = new_tokio_runtime("buck2-rt"); builder.enable_all(); - builder.thread_name("buck2-rt"); if let Some(threads) = buck2_env!("BUCK2_RUNTIME_THREADS", type=usize)? { builder.worker_threads(threads); @@ -322,9 +322,8 @@ impl DaemonCommand { let rt = builder.build().context("Error creating Tokio runtime")?; let handle = rt.handle().clone(); - let rt = Builder::new_multi_thread() + let rt = new_tokio_runtime("buck2-tn") .enable_all() - .thread_name("buck2-tn") // These values are arbitrary, but I/O shouldn't take up many threads. .worker_threads(2) .max_blocking_threads(2) @@ -386,15 +385,13 @@ impl DaemonCommand { let checker_interval_seconds = self.checker_interval_seconds; - thread::Builder::new() - .name("check-daemon-dir".to_owned()) - .spawn(move || { - Self::check_daemon_dir_thread( - checker_interval_seconds, - daemon_dir, - hard_shutdown_sender, - ) - })?; + thread_spawn("check-daemon-dir", move || { + Self::check_daemon_dir_thread( + checker_interval_seconds, + daemon_dir, + hard_shutdown_sender, + ) + })?; tracing::info!("Initialization complete, running the server."); diff --git a/app/buck2/src/commands/daemonize.rs b/app/buck2/src/commands/daemonize.rs index cf263bf2c5f35..dd1aa5c7db949 100644 --- a/app/buck2/src/commands/daemonize.rs +++ b/app/buck2/src/commands/daemonize.rs @@ -73,7 +73,6 @@ enum Outcome { /// * change root directory; /// * change the pid-file ownership to provided user (and/or) group; /// * execute any provided action just before dropping privileges. -/// pub(crate) struct Daemonize { stdin: Stdio, stdout: Stdio, diff --git a/app/buck2/src/commands/schedule_termination.rs b/app/buck2/src/commands/schedule_termination.rs index 76e5c893c70f5..6c4acfd349aaa 100644 --- a/app/buck2/src/commands/schedule_termination.rs +++ b/app/buck2/src/commands/schedule_termination.rs @@ -12,6 +12,7 @@ use std::time::Duration; use buck2_core::buck2_env; use buck2_util::process_stats::process_cpu_time_us; +use buck2_util::threads::thread_spawn; fn elapsed_cpu_time_as_percents( cpu_time_before_us: Option, @@ -29,34 +30,32 @@ fn elapsed_cpu_time_as_percents( /// if they are terminated. This allows the daemon to self-destruct. pub(crate) fn maybe_schedule_termination() -> anyhow::Result<()> { if let Some(duration) = buck2_env!("BUCK2_TERMINATE_AFTER", type=u64)? { - thread::Builder::new() - .name("buck2-terminate-after".to_owned()) - .spawn(move || { - const MEASURE_CPU_TIME_FOR: u64 = 10; - let (sleep_before, sleep_after) = match duration.checked_sub(MEASURE_CPU_TIME_FOR) { - Some(sleep_before) => (sleep_before, MEASURE_CPU_TIME_FOR), - None => (0, duration), - }; + thread_spawn("buck2-terminate-after", move || { + const MEASURE_CPU_TIME_FOR: u64 = 10; + let (sleep_before, sleep_after) = match duration.checked_sub(MEASURE_CPU_TIME_FOR) { + Some(sleep_before) => (sleep_before, MEASURE_CPU_TIME_FOR), + None => (0, duration), + }; - thread::sleep(Duration::from_secs(sleep_before)); - let process_cpu_time_us_before = process_cpu_time_us(); - thread::sleep(Duration::from_secs(sleep_after)); - let process_cpu_time_us_after = process_cpu_time_us(); + thread::sleep(Duration::from_secs(sleep_before)); + let process_cpu_time_us_before = process_cpu_time_us(); + thread::sleep(Duration::from_secs(sleep_after)); + let process_cpu_time_us_after = process_cpu_time_us(); - let elapsed_cpu_time_avg_in_percents = elapsed_cpu_time_as_percents( - process_cpu_time_us_before, - process_cpu_time_us_after, - sleep_after, + let elapsed_cpu_time_avg_in_percents = elapsed_cpu_time_as_percents( + process_cpu_time_us_before, + process_cpu_time_us_after, + sleep_after, + ); + if let Some(elapsed_cpu_time_avg_in_percents) = elapsed_cpu_time_avg_in_percents { + panic!( + "Buck is exiting after {}s elapsed; avg process CPU in the last {}s is {}%", + duration, sleep_after, elapsed_cpu_time_avg_in_percents ); - if let Some(elapsed_cpu_time_avg_in_percents) = elapsed_cpu_time_avg_in_percents { - panic!( - "Buck is exiting after {}s elapsed; avg process CPU in the last {}s is {}%", - duration, sleep_after, elapsed_cpu_time_avg_in_percents - ); - } else { - panic!("Buck is exiting after {}s elapsed", duration); - } - })?; + } else { + panic!("Buck is exiting after {}s elapsed", duration); + } + })?; } Ok(()) diff --git a/app/buck2/src/lib.rs b/app/buck2/src/lib.rs index b6ecdadd7263d..7f178510da97a 100644 --- a/app/buck2/src/lib.rs +++ b/app/buck2/src/lib.rs @@ -84,6 +84,9 @@ fn parse_isolation_dir(s: &str) -> anyhow::Result { /// Options of `buck2` command, before subcommand. #[derive(Clone, Debug, clap::Parser)] struct BeforeSubcommandOptions { + /// The name of the directory that Buck2 creates within buck-out for writing outputs and daemon + /// information. If one is not provided, Buck2 creates a directory with the default name. + /// /// Instances of Buck2 share a daemon if and only if their isolation directory is identical. /// The isolation directory also influences the output paths provided by Buck2, /// and as a result using a non-default isolation dir will cause cache misses (and slower builds). diff --git a/app/buck2/src/no_buckd.rs b/app/buck2/src/no_buckd.rs index f7a46b44c69ab..cd64d63d7f957 100644 --- a/app/buck2/src/no_buckd.rs +++ b/app/buck2/src/no_buckd.rs @@ -7,8 +7,6 @@ * of this source tree. */ -use std::thread; - use anyhow::Context; use buck2_client::commands::kill::kill_command_impl; use buck2_client_ctx::daemon::client::connect::buckd_startup_timeout; @@ -17,6 +15,7 @@ use buck2_client_ctx::startup_deadline::StartupDeadline; use buck2_common::invocation_paths::InvocationPaths; use buck2_common::legacy_configs::init::DaemonStartupConfig; use buck2_core::logging::LogConfigurationReloadHandle; +use buck2_util::threads::thread_spawn; use fbinit::FacebookInit; use crate::commands::daemon::DaemonCommand; @@ -49,7 +48,7 @@ pub(crate) fn start_in_process_daemon( Ok(Some(Box::new(move || { let (tx, rx) = std::sync::mpsc::channel(); // Spawn a thread which runs the daemon. - thread::spawn(move || { + thread_spawn("buck2-no-buckd", move || { let tx_clone = tx.clone(); let result = DaemonCommand::new_in_process(daemon_startup_config).exec( init, @@ -74,7 +73,7 @@ pub(crate) fn start_in_process_daemon( )), } } - }); + })?; // Wait for listener to start (or to fail). match rx.recv() { Ok(r) => r, diff --git a/app/buck2/src/panic.rs b/app/buck2/src/panic.rs index 4025af34da3fc..0ec345e16badc 100644 --- a/app/buck2/src/panic.rs +++ b/app/buck2/src/panic.rs @@ -54,7 +54,6 @@ fn the_panic_hook(fb: FacebookInit, info: &PanicInfo) { mod imp { use std::collections::HashMap; use std::panic::PanicInfo; - use std::thread; use std::time::Duration; use backtrace::Backtrace; @@ -63,6 +62,7 @@ mod imp { use buck2_events::metadata; use buck2_events::sink::scribe::new_thrift_scribe_sink_if_enabled; use buck2_events::BuckEvent; + use buck2_util::threads::thread_spawn; use fbinit::FacebookInit; use tokio::runtime::Builder; @@ -131,7 +131,7 @@ mod imp { map } - /// Writes a representation of the given `PanicInfo` to Scribe, via the `Panic` event. + /// Writes a representation of the given `PanicInfo` to Scribe, via the `StructuredError` event. pub(crate) fn write_panic_to_scribe(fb: FacebookInit, info: &PanicInfo) { let message = get_message_for_panic(info); let location = info.location().map(|loc| Location { @@ -238,23 +238,22 @@ mod imp { // on that thread. // // Note that if we fail to spawn a writer thread, then we just won't log. - let _err = thread::Builder::new() - .spawn(move || { - let runtime = Builder::new_current_thread().enable_all().build().unwrap(); - runtime.block_on( - sink.send_now(BuckEvent::new( - SystemTime::now(), - TraceId::new(), - None, - None, - InstantEvent { - data: Some(data.into()), - } - .into(), - )), - ); - }) - .map_err(|_| ()) - .and_then(|t| t.join().map_err(|_| ())); + let _err = thread_spawn("buck2-write-panic-to-scribe", move || { + let runtime = Builder::new_current_thread().enable_all().build().unwrap(); + runtime.block_on( + sink.send_now(BuckEvent::new( + SystemTime::now(), + TraceId::new(), + None, + None, + InstantEvent { + data: Some(data.into()), + } + .into(), + )), + ); + }) + .map_err(|_| ()) + .and_then(|t| t.join().map_err(|_| ())); } } diff --git a/app/buck2_action_impl/src/actions/impls/cas_artifact.rs b/app/buck2_action_impl/src/actions/impls/cas_artifact.rs index 576d0d41b62b4..342e781609f81 100644 --- a/app/buck2_action_impl/src/actions/impls/cas_artifact.rs +++ b/app/buck2_action_impl/src/actions/impls/cas_artifact.rs @@ -304,8 +304,7 @@ impl IncrementalActionExecutable for CasArtifactAction { .await?; let io_provider = ctx.io_provider(); - let maybe_io_tracer = io_provider.as_any().downcast_ref::(); - if let Some(tracer) = maybe_io_tracer { + if let Some(tracer) = TracingIoProvider::from_io(&*io_provider) { let offline_cache_path = offline::declare_copy_to_offline_output_cache(ctx, &self.output, value.dupe()) .await?; diff --git a/app/buck2_action_impl/src/actions/impls/download_file.rs b/app/buck2_action_impl/src/actions/impls/download_file.rs index 79d534099e71a..34ef894ba6bcc 100644 --- a/app/buck2_action_impl/src/actions/impls/download_file.rs +++ b/app/buck2_action_impl/src/actions/impls/download_file.rs @@ -330,8 +330,7 @@ impl IncrementalActionExecutable for DownloadFileAction { // If we're tracing I/O, get the materializer to copy to the offline cache // so we can include it in the offline archive manifest later. let io_provider = ctx.io_provider(); - let maybe_io_tracer = io_provider.as_any().downcast_ref::(); - if let Some(tracer) = maybe_io_tracer { + if let Some(tracer) = TracingIoProvider::from_io(&*io_provider) { let offline_cache_path = offline::declare_copy_to_offline_output_cache(ctx, self.output(), value.dupe()) .await?; diff --git a/app/buck2_action_impl/src/actions/impls/offline.rs b/app/buck2_action_impl/src/actions/impls/offline.rs index 2bb216a6565a8..d6ca515f2afe5 100644 --- a/app/buck2_action_impl/src/actions/impls/offline.rs +++ b/app/buck2_action_impl/src/actions/impls/offline.rs @@ -47,15 +47,12 @@ pub(crate) async fn declare_copy_from_offline_cache( .fs() .resolve_offline_output_cache_path(output.get_path()); - let (value, _hashing_time) = ctx - .blocking_executor() - .execute_io_inline(|| { - build_entry_from_disk( - ctx.fs().fs().resolve(&offline_cache_path), - FileDigestConfig::build(ctx.digest_config().cas_digest_config()), - ) - }) - .await?; + let (value, _hashing_time) = build_entry_from_disk( + ctx.fs().fs().resolve(&offline_cache_path), + FileDigestConfig::build(ctx.digest_config().cas_digest_config()), + ctx.blocking_executor(), + ) + .await?; let entry = value .ok_or_else(|| anyhow::anyhow!("Missing offline cache entry: `{}`", offline_cache_path))? diff --git a/app/buck2_action_impl/src/actions/impls/run/dep_files.rs b/app/buck2_action_impl/src/actions/impls/run/dep_files.rs index 2f981f77975ba..cf9fdc6a994d6 100644 --- a/app/buck2_action_impl/src/actions/impls/run/dep_files.rs +++ b/app/buck2_action_impl/src/actions/impls/run/dep_files.rs @@ -1344,7 +1344,7 @@ impl CommandLineArtifactVisitor for DepFilesCommandLineVisitor<'_> { } #[cfg(test)] -mod test { +mod tests { use buck2_artifact::artifact::artifact_type::testing::BuildArtifactTestingExt; use buck2_artifact::deferred::id::DeferredId; diff --git a/app/buck2_analysis/src/analysis/calculation.rs b/app/buck2_analysis/src/analysis/calculation.rs index 9d786ffeb89e1..824e9e79dfed3 100644 --- a/app/buck2_analysis/src/analysis/calculation.rs +++ b/app/buck2_analysis/src/analysis/calculation.rs @@ -23,6 +23,7 @@ use buck2_build_api::keep_going; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; +use buck2_data::error::ErrorTag; use buck2_data::ToProtoMessage; use buck2_error::Context; use buck2_events::dispatch::async_record_root_spans; @@ -34,6 +35,7 @@ use buck2_interpreter::starlark_profiler::StarlarkProfileDataAndStats; use buck2_interpreter::starlark_profiler::StarlarkProfileModeOrInstrumentation; use buck2_node::attrs::attr_type::query::ResolvedQueryLiterals; use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured::ConfiguredTargetNodeRef; use buck2_node::nodes::configured_frontend::ConfiguredTargetNodeCalculation; use buck2_node::rule_type::RuleType; use buck2_node::rule_type::StarlarkRuleType; @@ -50,9 +52,9 @@ use futures::StreamExt; use smallvec::SmallVec; use starlark::eval::ProfileMode; -use crate::analysis::env::get_user_defined_rule_impl; +use crate::analysis::env::get_user_defined_rule_spec; use crate::analysis::env::run_analysis; -use crate::analysis::env::RuleImplFunction; +use crate::analysis::env::RuleSpec; use crate::attrs::resolve::ctx::AnalysisQueryResult; #[derive(Debug, buck2_error::Error)] @@ -120,7 +122,7 @@ impl RuleAnalsysisCalculationImpl for RuleAnalysisCalculationInstance { pub async fn resolve_queries( ctx: &DiceComputations, - configured_node: &ConfiguredTargetNode, + configured_node: ConfiguredTargetNodeRef<'_>, ) -> anyhow::Result>> { let mut queries = configured_node.queries().peekable(); @@ -144,13 +146,12 @@ pub async fn resolve_queries( async fn resolve_queries_impl( ctx: &DiceComputations, - configured_node: &ConfiguredTargetNode, + configured_node: ConfiguredTargetNodeRef<'_>, queries: impl IntoIterator)>, ) -> anyhow::Result>> { let deps: TargetSet<_> = configured_node.deps().duped().collect(); let query_results = futures::future::try_join_all(queries.into_iter().map( |(query, resolved_literals_labels)| { - let ctx = ctx; let deps = &deps; async move { let mut resolved_literals = @@ -194,7 +195,7 @@ async fn resolve_queries_impl( } pub async fn get_dep_analysis<'v>( - configured_node: &'v ConfiguredTargetNode, + configured_node: ConfiguredTargetNodeRef<'v>, ctx: &DiceComputations, ) -> anyhow::Result> { keep_going::try_join_all( @@ -213,20 +214,30 @@ pub async fn get_dep_analysis<'v>( .await } -pub async fn get_rule_impl( +pub async fn get_rule_spec( ctx: &DiceComputations, func: &StarlarkRuleType, -) -> anyhow::Result { +) -> anyhow::Result { let module = ctx .get_loaded_module_from_import_path(&func.import_path) .await?; - Ok(get_user_defined_rule_impl(module.env().dupe(), func)) + Ok(get_user_defined_rule_spec(module.env().dupe(), func)) } async fn get_analysis_result( ctx: &DiceComputations, target: &ConfiguredTargetLabel, profile_mode: &StarlarkProfileModeOrInstrumentation, +) -> anyhow::Result> { + get_analysis_result_inner(ctx, target, profile_mode) + .await + .tag(ErrorTag::Analysis) +} + +async fn get_analysis_result_inner( + ctx: &DiceComputations, + target: &ConfiguredTargetLabel, + profile_mode: &StarlarkProfileModeOrInstrumentation, ) -> anyhow::Result> { let configured_node: MaybeCompatible = ctx.get_configured_target_node(target).await?; @@ -237,7 +248,7 @@ async fn get_analysis_result( MaybeCompatible::Compatible(configured_node) => configured_node, }; - let configured_node = &configured_node; + let configured_node = configured_node.as_ref(); let mut dep_analysis = get_dep_analysis(configured_node, ctx).await?; let now = Instant::now(); @@ -246,7 +257,7 @@ async fn get_analysis_result( let func = configured_node.rule_type(); match func { RuleType::Starlark(func) => { - let rule_impl = get_rule_impl(ctx, func).await?; + let rule_spec = get_rule_spec(ctx, func).await?; let start_event = buck2_data::AnalysisStart { target: Some(target.as_proto().into()), rule: func.to_string(), @@ -272,7 +283,7 @@ async fn get_analysis_result( dep_analysis, query_results, configured_node.execution_platform_resolution(), - &rule_impl, + &rule_spec, configured_node, profile_mode, ) diff --git a/app/buck2_analysis/src/analysis/env.rs b/app/buck2_analysis/src/analysis/env.rs index ffcf83bca5b17..36485df9e2b79 100644 --- a/app/buck2_analysis/src/analysis/env.rs +++ b/app/buck2_analysis/src/analysis/env.rs @@ -37,7 +37,7 @@ use buck2_interpreter::starlark_profiler::StarlarkProfilerOrInstrumentation; use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use buck2_interpreter::types::rule::FROZEN_PROMISE_ARTIFACT_MAPPINGS_GET_IMPL; use buck2_interpreter::types::rule::FROZEN_RULE_GET_IMPL; -use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured::ConfiguredTargetNodeRef; use buck2_node::rule_type::StarlarkRuleType; use dice::DiceComputations; use dupe::Dupe; @@ -160,7 +160,7 @@ pub fn resolve_query<'v>( } } -pub trait RuleImplFunction: Sync { +pub trait RuleSpec: Sync { fn invoke<'v>( &self, eval: &mut Evaluator<'v, '_>, @@ -175,7 +175,7 @@ pub trait RuleImplFunction: Sync { /// Container for the environment that analysis implementation functions should run in struct AnalysisEnv<'a> { - impl_function: &'a dyn RuleImplFunction, + rule_spec: &'a dyn RuleSpec, deps: HashMap<&'a ConfiguredTargetLabel, FrozenProviderCollectionValue>, query_results: HashMap>, execution_platform: &'a ExecutionPlatformResolution, @@ -188,17 +188,12 @@ pub(crate) async fn run_analysis<'a>( results: Vec<(&'a ConfiguredTargetLabel, AnalysisResult)>, query_results: HashMap>, execution_platform: &'a ExecutionPlatformResolution, - impl_function: &'a dyn RuleImplFunction, - node: &ConfiguredTargetNode, + rule_spec: &'a dyn RuleSpec, + node: ConfiguredTargetNodeRef<'_>, profile_mode: &StarlarkProfileModeOrInstrumentation, ) -> anyhow::Result { - let analysis_env = AnalysisEnv::new( - label, - results, - query_results, - execution_platform, - impl_function, - )?; + let analysis_env = + AnalysisEnv::new(label, results, query_results, execution_platform, rule_spec)?; run_analysis_with_env(dice, analysis_env, node, profile_mode).await } @@ -209,10 +204,10 @@ impl<'a> AnalysisEnv<'a> { results: Vec<(&'a ConfiguredTargetLabel, AnalysisResult)>, query_results: HashMap>, execution_platform: &'a ExecutionPlatformResolution, - impl_function: &'a dyn RuleImplFunction, + rule_spec: &'a dyn RuleSpec, ) -> anyhow::Result { Ok(AnalysisEnv { - impl_function, + rule_spec, deps: get_deps_from_analysis_results(results)?, query_results, execution_platform, @@ -233,7 +228,7 @@ pub fn get_deps_from_analysis_results<'v>( fn run_analysis_with_env<'a>( dice: &'a DiceComputations, analysis_env: AnalysisEnv<'a>, - node: &'a ConfiguredTargetNode, + node: ConfiguredTargetNodeRef<'a>, profile_mode: &'a StarlarkProfileModeOrInstrumentation, ) -> impl Future> + Send + 'a { let fut = async move { @@ -245,7 +240,7 @@ fn run_analysis_with_env<'a>( async fn run_analysis_with_env_underlying( dice: &DiceComputations, analysis_env: AnalysisEnv<'_>, - node: &ConfiguredTargetNode, + node: ConfiguredTargetNodeRef<'_>, profile_mode: &StarlarkProfileModeOrInstrumentation, ) -> anyhow::Result { let env = Module::new(); @@ -304,7 +299,7 @@ async fn run_analysis_with_env_underlying( dice.global_data().get_digest_config(), )); - let list_res = analysis_env.impl_function.invoke(&mut eval, ctx)?; + let list_res = analysis_env.rule_spec.invoke(&mut eval, ctx)?; // TODO(cjhopman): This seems quite wrong. This should be happening after run_promises. provider @@ -361,16 +356,16 @@ async fn run_analysis_with_env_underlying( )) } -pub fn get_user_defined_rule_impl( +pub fn get_user_defined_rule_spec( module: FrozenModule, rule_type: &StarlarkRuleType, -) -> impl RuleImplFunction { +) -> impl RuleSpec { struct Impl { module: FrozenModule, name: String, } - impl RuleImplFunction for Impl { + impl RuleSpec for Impl { fn invoke<'v>( &self, eval: &mut Evaluator<'v, '_>, diff --git a/app/buck2_analysis/src/analysis/plugins.rs b/app/buck2_analysis/src/analysis/plugins.rs index 8c5cb85266d82..53a4e072f7f20 100644 --- a/app/buck2_analysis/src/analysis/plugins.rs +++ b/app/buck2_analysis/src/analysis/plugins.rs @@ -12,7 +12,7 @@ use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_node::attrs::attr_type::dep::DepAttr; use buck2_node::attrs::attr_type::dep::DepAttrTransition; use buck2_node::attrs::attr_type::dep::DepAttrType; -use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured::ConfiguredTargetNodeRef; use buck2_node::provider_id_set::ProviderIdSet; use dupe::IterDupedExt; use starlark::values::Value; @@ -23,7 +23,7 @@ use crate::attrs::resolve::attr_type::dep::DepAttrTypeExt; use crate::attrs::resolve::ctx::AttrResolutionContext; pub fn plugins_to_starlark_value<'v>( - node: &ConfiguredTargetNode, + node: ConfiguredTargetNodeRef, ctx: &dyn AttrResolutionContext<'v>, ) -> anyhow::Result>> { let mut plugins = SmallMap::new(); diff --git a/app/buck2_analysis/src/attrs/resolve/node_to_attrs_struct.rs b/app/buck2_analysis/src/attrs/resolve/node_to_attrs_struct.rs index 6eee0b1ee7291..26cc5e1a5ae56 100644 --- a/app/buck2_analysis/src/attrs/resolve/node_to_attrs_struct.rs +++ b/app/buck2_analysis/src/attrs/resolve/node_to_attrs_struct.rs @@ -8,7 +8,7 @@ */ use buck2_node::attrs::inspect_options::AttrInspectOptions; -use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured::ConfiguredTargetNodeRef; use starlark::values::structs::AllocStruct; use starlark::values::Value; @@ -17,7 +17,7 @@ use crate::attrs::resolve::ctx::AttrResolutionContext; /// Prepare `ctx.attrs` for rule impl. pub(crate) fn node_to_attrs_struct<'v>( - node: &ConfiguredTargetNode, + node: ConfiguredTargetNodeRef, ctx: &dyn AttrResolutionContext<'v>, ) -> anyhow::Result> { let attrs_iter = node.attrs(AttrInspectOptions::All); diff --git a/app/buck2_anon_target/src/anon_target_attr.rs b/app/buck2_anon_target/src/anon_target_attr.rs index 7c963db8865f1..b9e9b5499767f 100644 --- a/app/buck2_anon_target/src/anon_target_attr.rs +++ b/app/buck2_anon_target/src/anon_target_attr.rs @@ -12,7 +12,6 @@ use std::fmt::Display; use allocative::Allocative; use buck2_artifact::artifact::artifact_type::Artifact; -use buck2_build_api::interpreter::rule_defs::artifact::StarlarkPromiseArtifact; use buck2_core::package::PackageLabel; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersLabel; @@ -37,6 +36,9 @@ use serde::Serialize; use serde::Serializer; use serde_json::to_value; +use crate::anon_target_attr_resolve::AnonTargetAttrTraversal; +use crate::promise_artifacts::PromiseArtifactAttr; + #[derive(Debug, Clone, PartialEq, Eq, Hash, Allocative)] pub enum AnonTargetAttr { Bool(BoolLiteral), @@ -64,7 +66,7 @@ pub enum AnonTargetAttr { // Accepts any bound artifacts. Maps to `attr.source()`. Artifact(Artifact), // Accepts unresolved promise artifacts. Maps to `attr.source()`. - PromiseArtifact(StarlarkPromiseArtifact), + PromiseArtifact(PromiseArtifactAttr), Arg(ConfiguredStringWithMacros), Label(ProvidersLabel), } @@ -178,6 +180,47 @@ impl AnonTargetAttr { } } + #[allow(unused)] + pub fn traverse_anon_attr<'a>( + &'a self, + traversal: &mut dyn AnonTargetAttrTraversal, + ) -> anyhow::Result<()> { + match self { + AnonTargetAttr::Bool(_) => Ok(()), + AnonTargetAttr::Int(_) => Ok(()), + AnonTargetAttr::String(_) => Ok(()), + AnonTargetAttr::EnumVariant(_) => Ok(()), + AnonTargetAttr::List(list) => { + for v in list.iter() { + v.traverse_anon_attr(traversal)?; + } + Ok(()) + } + AnonTargetAttr::Tuple(list) => { + for v in list.iter() { + v.traverse_anon_attr(traversal)?; + } + Ok(()) + } + AnonTargetAttr::Dict(dict) => { + for (k, v) in dict.iter() { + k.traverse_anon_attr(traversal)?; + v.traverse_anon_attr(traversal)?; + } + Ok(()) + } + AnonTargetAttr::None => Ok(()), + AnonTargetAttr::OneOf(l, _) => l.traverse_anon_attr(traversal), + AnonTargetAttr::Dep(_) => Ok(()), + AnonTargetAttr::Artifact(_) => Ok(()), + AnonTargetAttr::Arg(_) => Ok(()), + AnonTargetAttr::PromiseArtifact(promise_artifact) => { + traversal.promise_artifact(promise_artifact) + } + AnonTargetAttr::Label(_) => Ok(()), + } + } + pub fn _unpack_list(&self) -> Option<&[AnonTargetAttr]> { match self { AnonTargetAttr::List(list) => Some(list), diff --git a/app/buck2_anon_target/src/anon_target_attr_coerce.rs b/app/buck2_anon_target/src/anon_target_attr_coerce.rs index de86d561e8512..c752a3952612f 100644 --- a/app/buck2_anon_target/src/anon_target_attr_coerce.rs +++ b/app/buck2_anon_target/src/anon_target_attr_coerce.rs @@ -46,6 +46,7 @@ use starlark::values::Value; use crate::anon_target_attr::AnonTargetAttr; use crate::anon_targets::AnonAttrCtx; +use crate::promise_artifacts::PromiseArtifactAttr; pub trait AnonTargetAttrTypeCoerce { fn coerce_item(&self, ctx: &AnonAttrCtx, value: Value) -> anyhow::Result; @@ -134,7 +135,10 @@ impl AnonTargetAttrTypeCoerce for AttrType { // Check if this is a StarlarkPromiseArtifact first before checking other artifact types to // allow anon targets to accept unresolved promise artifacts. if let Some(promise_artifact) = StarlarkPromiseArtifact::from_value(value) { - Ok(AnonTargetAttr::PromiseArtifact(promise_artifact.clone())) + Ok(AnonTargetAttr::PromiseArtifact(PromiseArtifactAttr { + id: promise_artifact.artifact.id.as_ref().clone(), + short_path: promise_artifact.short_path.clone(), + })) } else if let Some(artifact_like) = ValueAsArtifactLike::unpack_value(value) { let artifact = artifact_like.0.get_bound_artifact()?; Ok(AnonTargetAttr::Artifact(artifact)) diff --git a/app/buck2_anon_target/src/anon_target_attr_resolve.rs b/app/buck2_anon_target/src/anon_target_attr_resolve.rs index 021b79251a862..2030f0638cf4f 100644 --- a/app/buck2_anon_target/src/anon_target_attr_resolve.rs +++ b/app/buck2_anon_target/src/anon_target_attr_resolve.rs @@ -7,37 +7,66 @@ * of this source tree. */ +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::OnceLock; + +use buck2_analysis::analysis::env::RuleAnalysisAttrResolutionContext; use buck2_analysis::attrs::resolve::attr_type::arg::ConfiguredStringWithMacrosExt; use buck2_analysis::attrs::resolve::attr_type::dep::DepAttrTypeExt; use buck2_analysis::attrs::resolve::ctx::AttrResolutionContext; +use buck2_artifact::artifact::artifact_type::Artifact; +use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; +use buck2_build_api::artifact_groups::promise::PromiseArtifact; +use buck2_build_api::artifact_groups::promise::PromiseArtifactResolveError; use buck2_build_api::interpreter::rule_defs::artifact::StarlarkArtifact; +use buck2_build_api::interpreter::rule_defs::artifact::StarlarkPromiseArtifact; +use buck2_build_api::interpreter::rule_defs::provider::collection::FrozenProviderCollectionValue; +use buck2_build_api::keep_going; use buck2_core::package::PackageLabel; +use buck2_core::provider::label::ConfiguredProvidersLabel; +use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_interpreter::error::BuckStarlarkError; use buck2_interpreter::types::configured_providers_label::StarlarkProvidersLabel; use buck2_node::attrs::attr_type::dep::DepAttrType; +use buck2_node::attrs::attr_type::query::ResolvedQueryLiterals; +use buck2_node::attrs::configured_traversal::ConfiguredAttrTraversal; +use dice::DiceComputations; use dupe::Dupe; +use futures::stream::FuturesUnordered; use starlark::values::dict::Dict; use starlark::values::tuple::AllocTuple; use starlark::values::Value; use starlark_map::small_map::SmallMap; use crate::anon_target_attr::AnonTargetAttr; +use crate::anon_targets::get_artifact_from_anon_target_analysis; +use crate::anon_targets::AnonTargetKey; +use crate::anon_targets::AnonTargetsError; +use crate::promise_artifacts::PromiseArtifactAttr; + +// No macros in anon targets, so query results are empty. Execution platform resolution should +// always be inherited from the anon target. +pub(crate) struct AnonTargetAttrResolutionContext<'v> { + pub(crate) promised_artifacts_map: HashMap<&'v PromiseArtifactAttr, Artifact>, + pub(crate) rule_analysis_attr_resolution_ctx: RuleAnalysisAttrResolutionContext<'v>, +} -pub trait AnonTargetAttrExt { +pub trait AnonTargetAttrResolution { fn resolve<'v>( &self, pkg: PackageLabel, - ctx: &dyn AttrResolutionContext<'v>, + ctx: &AnonTargetAttrResolutionContext<'v>, ) -> anyhow::Result>>; fn resolve_single<'v>( &self, pkg: PackageLabel, - ctx: &dyn AttrResolutionContext<'v>, + ctx: &AnonTargetAttrResolutionContext<'v>, ) -> anyhow::Result>; } -impl AnonTargetAttrExt for AnonTargetAttr { +impl AnonTargetAttrResolution for AnonTargetAttr { /// "Resolves" the anon target attr value to the resolved value provided to the rule implementation. /// /// `resolve` may return multiple values. It is up to the caller to fail if @@ -47,7 +76,7 @@ impl AnonTargetAttrExt for AnonTargetAttr { fn resolve<'v>( &self, pkg: PackageLabel, - ctx: &dyn AttrResolutionContext<'v>, + ctx: &AnonTargetAttrResolutionContext<'v>, ) -> anyhow::Result>> { Ok(vec![self.resolve_single(pkg, ctx)?]) } @@ -57,8 +86,9 @@ impl AnonTargetAttrExt for AnonTargetAttr { fn resolve_single<'v>( &self, pkg: PackageLabel, - ctx: &dyn AttrResolutionContext<'v>, + anon_resolution_ctx: &AnonTargetAttrResolutionContext<'v>, ) -> anyhow::Result> { + let ctx = &anon_resolution_ctx.rule_analysis_attr_resolution_ctx; match self { AnonTargetAttr::Bool(v) => Ok(Value::new_bool(v.0)), AnonTargetAttr::Int(v) => Ok(ctx.heap().alloc(*v)), @@ -68,14 +98,14 @@ impl AnonTargetAttrExt for AnonTargetAttr { AnonTargetAttr::List(list) => { let mut values = Vec::with_capacity(list.len()); for v in list.iter() { - values.append(&mut v.resolve(pkg.dupe(), ctx)?); + values.append(&mut v.resolve(pkg.dupe(), anon_resolution_ctx)?); } Ok(ctx.heap().alloc(values)) } AnonTargetAttr::Tuple(list) => { let mut values = Vec::with_capacity(list.len()); for v in list.iter() { - values.append(&mut v.resolve(pkg.dupe(), ctx)?); + values.append(&mut v.resolve(pkg.dupe(), anon_resolution_ctx)?); } Ok(ctx.heap().alloc(AllocTuple(values))) } @@ -83,23 +113,162 @@ impl AnonTargetAttrExt for AnonTargetAttr { let mut res = SmallMap::with_capacity(dict.len()); for (k, v) in dict.iter() { res.insert_hashed( - k.resolve_single(pkg.dupe(), ctx)? + k.resolve_single(pkg.dupe(), anon_resolution_ctx)? .get_hashed() .map_err(BuckStarlarkError::new)?, - v.resolve_single(pkg.dupe(), ctx)?, + v.resolve_single(pkg.dupe(), anon_resolution_ctx)?, ); } Ok(ctx.heap().alloc(Dict::new(res))) } AnonTargetAttr::None => Ok(Value::new_none()), - AnonTargetAttr::OneOf(box l, _) => l.resolve_single(pkg, ctx), + AnonTargetAttr::OneOf(box l, _) => l.resolve_single(pkg, anon_resolution_ctx), AnonTargetAttr::Dep(d) => DepAttrType::resolve_single(ctx, d), AnonTargetAttr::Artifact(d) => Ok(ctx.heap().alloc(StarlarkArtifact::new(d.clone()))), AnonTargetAttr::Arg(a) => a.resolve(ctx, &pkg), - AnonTargetAttr::PromiseArtifact(artifact) => Ok(ctx.heap().alloc(artifact.clone())), + AnonTargetAttr::PromiseArtifact(promise_artifact_attr) => { + let promise_id = promise_artifact_attr.id.clone(); + // We validated that the analysis contains the promise artifact id earlier + let artifact = anon_resolution_ctx + .promised_artifacts_map + .get(&promise_artifact_attr) + .unwrap(); + + // Assert the short path, since we have the real artifact now + if let Some(expected_short_path) = &promise_artifact_attr.short_path { + artifact.get_path().with_short_path(|artifact_short_path| { + if artifact_short_path != expected_short_path { + Err(anyhow::Error::from( + PromiseArtifactResolveError::ShortPathMismatch( + expected_short_path.clone(), + artifact_short_path.to_string(), + ), + )) + } else { + Ok(()) + } + })?; + } + + let fulfilled = OnceLock::new(); + fulfilled.set(artifact.clone()).unwrap(); + + let fulfilled_promise_inner = + PromiseArtifact::new(Arc::new(fulfilled), Arc::new(promise_id)); + + let fulfilled_promise_artifact = StarlarkPromiseArtifact::new( + None, + fulfilled_promise_inner, + promise_artifact_attr.short_path.clone(), + ); + + // To resolve the promise artifact attr, we end up creating a new `StarlarkPromiseArtifact` with the `OnceLock` set + // with the artifact that was found from the upstream analysis. + Ok(ctx.heap().alloc(fulfilled_promise_artifact)) + } AnonTargetAttr::Label(label) => { Ok(ctx.heap().alloc(StarlarkProvidersLabel::new(label.clone()))) } } } } + +// Container for things that require looking up analysis results in order to resolve the attribute. +pub(crate) struct AnonTargetDependents { + pub(crate) deps: Vec, + pub(crate) promise_artifacts: Vec, +} + +// Container for analysis results of the anon target dependents. +pub(crate) struct AnonTargetDependentAnalysisResults<'v> { + pub(crate) dep_analysis_results: + HashMap<&'v ConfiguredTargetLabel, FrozenProviderCollectionValue>, + pub(crate) promised_artifacts: HashMap<&'v PromiseArtifactAttr, Artifact>, +} + +pub(crate) trait AnonTargetAttrTraversal { + fn promise_artifact(&mut self, promise_artifact: &PromiseArtifactAttr) -> anyhow::Result<()>; +} + +impl AnonTargetDependents { + pub(crate) fn get_dependents( + anon_target: &AnonTargetKey, + ) -> anyhow::Result { + struct DepTraversal(Vec); + struct PromiseArtifactTraversal(Vec); + + impl ConfiguredAttrTraversal for DepTraversal { + fn dep(&mut self, dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { + self.0.push(dep.target().dupe()); + Ok(()) + } + + fn query( + &mut self, + _query: &str, + _resolved_literals: &ResolvedQueryLiterals, + ) -> anyhow::Result<()> { + Err(AnonTargetsError::QueryMacroNotSupported.into()) + } + } + + impl AnonTargetAttrTraversal for PromiseArtifactTraversal { + fn promise_artifact( + &mut self, + promise_artifact: &PromiseArtifactAttr, + ) -> anyhow::Result<()> { + self.0.push(promise_artifact.clone()); + Ok(()) + } + } + + let mut dep_traversal = DepTraversal(Vec::new()); + let mut promise_artifact_traversal = PromiseArtifactTraversal(Vec::new()); + for x in anon_target.0.attrs().values() { + x.traverse(anon_target.0.name().pkg(), &mut dep_traversal)?; + x.traverse_anon_attr(&mut promise_artifact_traversal)?; + } + Ok(AnonTargetDependents { + deps: dep_traversal.0, + promise_artifacts: promise_artifact_traversal.0, + }) + } + + pub(crate) async fn get_analysis_results<'v>( + &'v self, + dice: &'v DiceComputations, + ) -> anyhow::Result> { + let dep_analysis_results: HashMap<_, _> = keep_going::try_join_all( + dice, + self.deps + .iter() + .map(async move |dep| { + let res = dice + .get_analysis_result(dep) + .await + .and_then(|v| v.require_compatible()); + res.map(|x| (dep, x.providers().dupe())) + }) + .collect::>(), + ) + .await?; + + let promised_artifacts: HashMap<_, _> = keep_going::try_join_all( + dice, + self.promise_artifacts + .iter() + .map(async move |promise_artifact_attr| { + get_artifact_from_anon_target_analysis(&promise_artifact_attr.id, dice) + .await + .map(|artifact| (promise_artifact_attr, artifact)) + }) + .collect::>(), + ) + .await?; + + Ok(AnonTargetDependentAnalysisResults { + dep_analysis_results, + promised_artifacts, + }) + } +} diff --git a/app/buck2_anon_target/src/anon_targets.rs b/app/buck2_anon_target/src/anon_targets.rs index e7b884acae901..3b3078c0195b0 100644 --- a/app/buck2_anon_target/src/anon_targets.rs +++ b/app/buck2_anon_target/src/anon_targets.rs @@ -15,27 +15,26 @@ use std::sync::Arc; use allocative::Allocative; use anyhow::Context as _; use async_trait::async_trait; -use buck2_analysis::analysis::calculation::get_rule_impl; +use buck2_analysis::analysis::calculation::get_rule_spec; use buck2_analysis::analysis::env::RuleAnalysisAttrResolutionContext; -use buck2_analysis::analysis::env::RuleImplFunction; +use buck2_analysis::analysis::env::RuleSpec; use buck2_artifact::artifact::artifact_type::Artifact; use buck2_build_api::analysis::anon_promises_dyn::AnonPromisesDyn; use buck2_build_api::analysis::anon_targets_registry::AnonTargetsRegistryDyn; use buck2_build_api::analysis::anon_targets_registry::ANON_TARGET_REGISTRY_NEW; -use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; use buck2_build_api::analysis::registry::AnalysisRegistry; use buck2_build_api::analysis::AnalysisResult; use buck2_build_api::artifact_groups::promise::PromiseArtifact; use buck2_build_api::artifact_groups::promise::PromiseArtifactId; use buck2_build_api::artifact_groups::promise::PromiseArtifactResolveError; use buck2_build_api::deferred::calculation::EVAL_ANON_TARGET; +use buck2_build_api::deferred::calculation::GET_PROMISED_ARTIFACT; use buck2_build_api::deferred::types::DeferredTable; use buck2_build_api::interpreter::rule_defs::artifact::ValueAsArtifactLike; use buck2_build_api::interpreter::rule_defs::context::AnalysisContext; use buck2_build_api::interpreter::rule_defs::plugins::AnalysisPlugins; use buck2_build_api::interpreter::rule_defs::provider::collection::FrozenProviderCollectionValue; use buck2_build_api::interpreter::rule_defs::provider::collection::ProviderCollection; -use buck2_build_api::keep_going; use buck2_configured::nodes::calculation::find_execution_platform_by_configuration; use buck2_core::base_deferred_key::BaseDeferredKey; use buck2_core::base_deferred_key::BaseDeferredKeyDyn; @@ -56,7 +55,6 @@ use buck2_core::pattern::PatternData; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersLabel; use buck2_core::provider::label::ProvidersName; -use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_core::target::label::TargetLabel; use buck2_core::target::name::TargetNameRef; use buck2_core::unsafe_send_future::UnsafeSendFuture; @@ -71,13 +69,11 @@ use buck2_interpreter::starlark_profiler::StarlarkProfilerOrInstrumentation; use buck2_interpreter::starlark_promise::StarlarkPromise; use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use buck2_interpreter_for_build::rule::FrozenRuleCallable; -use buck2_node::attrs::attr_type::query::ResolvedQueryLiterals; use buck2_node::attrs::attr_type::AttrType; use buck2_node::attrs::coerced_attr::CoercedAttr; use buck2_node::attrs::coerced_path::CoercedPath; use buck2_node::attrs::coercion_context::AttrCoercionContext; use buck2_node::attrs::configuration_context::AttrConfigurationContext; -use buck2_node::attrs::configured_traversal::ConfiguredAttrTraversal; use buck2_node::attrs::internal::internal_attrs; use buck2_util::arc_str::ArcSlice; use buck2_util::arc_str::ArcStr; @@ -85,7 +81,6 @@ use derive_more::Display; use dice::DiceComputations; use dice::Key; use dupe::Dupe; -use futures::stream::FuturesUnordered; use futures::Future; use futures::FutureExt; use starlark::any::AnyLifetime; @@ -105,7 +100,9 @@ use starlark_map::small_map::SmallMap; use crate::anon_promises::AnonPromises; use crate::anon_target_attr::AnonTargetAttr; use crate::anon_target_attr_coerce::AnonTargetAttrTypeCoerce; -use crate::anon_target_attr_resolve::AnonTargetAttrExt; +use crate::anon_target_attr_resolve::AnonTargetAttrResolution; +use crate::anon_target_attr_resolve::AnonTargetAttrResolutionContext; +use crate::anon_target_attr_resolve::AnonTargetDependents; use crate::anon_target_node::AnonTarget; use crate::promise_artifacts::PromiseArtifactRegistry; @@ -298,45 +295,9 @@ impl AnonTargetKey { unsafe { UnsafeSendFuture::new_encapsulates_starlark(fut) } } - fn deps(&self) -> anyhow::Result> { - struct Traversal(Vec); - - impl ConfiguredAttrTraversal for Traversal { - fn dep(&mut self, dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { - self.0.push(dep.target().dupe()); - Ok(()) - } - - fn query( - &mut self, - _query: &str, - _resolved_literals: &ResolvedQueryLiterals, - ) -> anyhow::Result<()> { - Err(AnonTargetsError::QueryMacroNotSupported.into()) - } - } - - let mut traversal = Traversal(Vec::new()); - for x in self.0.attrs().values() { - x.traverse(self.0.name().pkg(), &mut traversal)?; - } - Ok(traversal.0) - } async fn run_analysis_impl(&self, dice: &DiceComputations) -> anyhow::Result { - let deps = self.deps()?; - let dep_analysis_results: HashMap<_, _> = keep_going::try_join_all( - dice, - deps.iter() - .map(async move |dep| { - let res = dice - .get_analysis_result(dep) - .await - .and_then(|v| v.require_compatible()); - res.map(|x| (dep, x.providers().dupe())) - }) - .collect::>(), - ) - .await?; + let dependents = AnonTargetDependents::get_dependents(self)?; + let dependents_analyses = dependents.get_analysis_results(dice).await?; let exec_resolution = ExecutionPlatformResolution::new( Some( @@ -350,7 +311,7 @@ impl AnonTargetKey { Vec::new(), ); - let rule_impl = get_rule_impl(dice, self.0.rule_type()).await?; + let rule_impl = get_rule_spec(dice, self.0.rule_type()).await?; let env = Module::new(); let print = EventDispatcherPrintHandler(get_dispatcher()); @@ -369,13 +330,18 @@ impl AnonTargetKey { eval.set_print_handler(&print); // No attributes are allowed to contain macros or other stuff, so an empty resolution context works - let resolution_ctx = RuleAnalysisAttrResolutionContext { + let rule_analysis_attr_resolution_ctx = RuleAnalysisAttrResolutionContext { module: &env, - dep_analysis_results, + dep_analysis_results: dependents_analyses.dep_analysis_results, query_results: HashMap::new(), execution_platform_resolution: exec_resolution.clone(), }; + let resolution_ctx = AnonTargetAttrResolutionContext { + promised_artifacts_map: dependents_analyses.promised_artifacts, + rule_analysis_attr_resolution_ctx, + }; + let mut resolved_attrs = Vec::with_capacity(self.0.attrs().len()); for (name, attr) in self.0.attrs().iter() { resolved_attrs.push(( @@ -473,7 +439,9 @@ impl AnonTargetKey { let mut fulfilled_artifact_mappings = HashMap::new(); for (id, func) in promise_artifact_mappings.values().enumerate() { - let artifact = eval_starlark_function(eval, *func, anon_target_result)?; + let artifact = eval + .eval_function(*func, &[anon_target_result], &[]) + .map_err(BuckStarlarkError::new)?; let promise_id = PromiseArtifactId::new(BaseDeferredKey::AnonTarget(self.0.clone()), id); @@ -484,11 +452,9 @@ impl AnonTargetKey { .insert(promise_id.clone(), artifact.0.get_bound_artifact()?); } None => { - return Err(PromiseArtifactResolveError::NotAnArtifact( - None, // TODO(@wendyy) remove declaration location from this error variant - artifact.to_repr(), - ) - .into()); + return Err( + PromiseArtifactResolveError::NotAnArtifact(artifact.to_repr()).into(), + ); } } } @@ -497,16 +463,6 @@ impl AnonTargetKey { } } -// TODO(@wendyy) - should put this somewhere common and reuse. -fn eval_starlark_function<'v>( - eval: &mut Evaluator<'v, '_>, - func: Value<'v>, - res: Value<'v>, -) -> anyhow::Result> { - eval.eval_function(func, &[res], &[]) - .map_err(|e| BuckStarlarkError::new(e).into()) -} - /// Several attribute functions need a context, make one that is mostly useless. pub struct AnonAttrCtx { cfg: ConfigurationData, @@ -607,6 +563,43 @@ pub(crate) fn init_eval_anon_target() { .init(|ctx, key| Box::pin(async move { AnonTargetKey::downcast(key)?.resolve(ctx).await })); } +pub(crate) fn init_get_promised_artifact() { + GET_PROMISED_ARTIFACT.init(|promise_artifact, ctx| { + Box::pin( + async move { get_artifact_from_anon_target_analysis(promise_artifact.id(), ctx).await }, + ) + }); +} + +pub(crate) async fn get_artifact_from_anon_target_analysis<'v>( + promise_id: &'v PromiseArtifactId, + ctx: &'v DiceComputations, +) -> anyhow::Result { + let owner = promise_id.owner(); + let analysis_result = match owner { + BaseDeferredKey::AnonTarget(anon_target) => { + AnonTargetKey::downcast(anon_target.dupe())? + .resolve(ctx) + .await? + } + _ => { + return Err(PromiseArtifactResolveError::OwnerIsNotAnonTarget( + promise_id.clone(), + owner.clone(), + ) + .into()); + } + }; + + analysis_result + .promise_artifact_map() + .get(promise_id) + .context(PromiseArtifactResolveError::NotFoundInAnalysis( + promise_id.clone(), + )) + .cloned() +} + pub(crate) fn init_anon_target_registry_new() { ANON_TARGET_REGISTRY_NEW.init(|_phantom, execution_platform| { Box::new(AnonTargetsRegistry { @@ -704,7 +697,7 @@ impl<'v> AnonTargetsRegistryDyn<'v> for AnonTargetsRegistry<'v> { } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_anon_target/src/lib.rs b/app/buck2_anon_target/src/lib.rs index f0fcacf776211..938d451741ecc 100644 --- a/app/buck2_anon_target/src/lib.rs +++ b/app/buck2_anon_target/src/lib.rs @@ -27,6 +27,7 @@ pub fn init_late_bindings() { ONCE.call_once(|| { anon_targets::init_anon_target_registry_new(); anon_targets::init_eval_anon_target(); + anon_targets::init_get_promised_artifact(); starlark_defs::init_analysis_actions_methods_anon_target(); starlark_defs::init_register_anon_target_types(); }); diff --git a/app/buck2_anon_target/src/promise_artifacts.rs b/app/buck2_anon_target/src/promise_artifacts.rs index 72cd9afc63cee..43f71e04a7199 100644 --- a/app/buck2_anon_target/src/promise_artifacts.rs +++ b/app/buck2_anon_target/src/promise_artifacts.rs @@ -7,6 +7,7 @@ * of this source tree. */ +use std::fmt; use std::fmt::Debug; use std::sync::Arc; use std::sync::OnceLock; @@ -14,6 +15,7 @@ use std::sync::OnceLock; use allocative::Allocative; use buck2_build_api::artifact_groups::promise::PromiseArtifact; use buck2_build_api::artifact_groups::promise::PromiseArtifactId; +use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; use dupe::Dupe; use gazebo::prelude::SliceExt; use starlark::codemap::FileSpan; @@ -65,3 +67,27 @@ impl PromiseArtifactRegistry { Ok(artifact) } } + +// When passing promise artifacts into anon targets, we will coerce them into this type. +// During resolve, we look up the analysis of the target that produced the promise artifact, +// assert short paths, and produce a new `StarlarkPromiseArtifact` with the `OnceLock` resolved. +#[allow(unused)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Allocative)] +pub(crate) struct PromiseArtifactAttr { + pub(crate) id: PromiseArtifactId, + pub(crate) short_path: Option, +} + +impl fmt::Display for PromiseArtifactAttr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO(@wendyy) - we should figure out what to do about the declaration location. + // It's possible that 2 targets produce the same promise artifact and try to pass + // it into a downstream target, so then there would be 2 declaration locations. + write!(f, "")?; + Ok(()) + } +} diff --git a/app/buck2_artifact/src/artifact/artifact_dump.rs b/app/buck2_artifact/src/artifact/artifact_dump.rs index cd145b81d8fba..12db47f578e6c 100644 --- a/app/buck2_artifact/src/artifact/artifact_dump.rs +++ b/app/buck2_artifact/src/artifact/artifact_dump.rs @@ -70,7 +70,7 @@ pub struct ArtifactMetadataJson<'a> { } #[cfg(test)] -mod test { +mod tests { use buck2_common::cas_digest::CasDigestConfig; use super::*; diff --git a/app/buck2_audit_server/src/analysis_queries.rs b/app/buck2_audit_server/src/analysis_queries.rs index 5cd0538d632ce..505858916b858 100644 --- a/app/buck2_audit_server/src/analysis_queries.rs +++ b/app/buck2_audit_server/src/analysis_queries.rs @@ -23,8 +23,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use dupe::Dupe; use gazebo::prelude::*; @@ -42,8 +42,9 @@ impl AuditSubcommand for AuditAnalysisQueriesCommand { .with_dice_ctx(async move |server_ctx, mut ctx| { let cells = ctx.get_cell_resolver().await?; - let global_target_platform = - target_platform_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(&client_ctx, server_ctx, &mut ctx) + .await?; let parsed_patterns = parse_patterns_from_cli_args::( &mut ctx, @@ -64,12 +65,12 @@ impl AuditSubcommand for AuditAnalysisQueriesCommand { for (target, TargetPatternExtra) in targets { let label = TargetLabel::new(package.dupe(), target.as_ref()); let configured_target = ctx - .get_configured_target(&label, global_target_platform.as_ref()) + .get_configured_target(&label, &global_cfg_options) .await?; let node = ctx.get_configured_target_node(&configured_target).await?; let node = node.require_compatible()?; - let query_results = resolve_queries(&ctx, &node).await?; + let query_results = resolve_queries(&ctx, node.as_ref()).await?; writeln!(stdout, "{}:", label)?; for (query, result) in &query_results { writeln!(stdout, " {}", query)?; diff --git a/app/buck2_audit_server/src/classpath.rs b/app/buck2_audit_server/src/classpath.rs index 2d5139c2048cb..32b250c8fdd81 100644 --- a/app/buck2_audit_server/src/classpath.rs +++ b/app/buck2_audit_server/src/classpath.rs @@ -20,8 +20,8 @@ use buck2_node::load_patterns::MissingTargetBehavior; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use dupe::Dupe; use gazebo::prelude::SliceExt; use indexmap::IndexMap; @@ -47,13 +47,14 @@ impl AuditSubcommand for AuditClasspathCommand { cwd, ) .await?; - let target_platform = - target_platform_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(&client_ctx, server_ctx, &mut ctx) + .await?; // Incompatible targets are skipped because this is an audit command let targets = load_compatible_patterns( &ctx, parsed_patterns, - target_platform, + &global_cfg_options, MissingTargetBehavior::Fail, ) .await?; diff --git a/app/buck2_audit_server/src/dep_files.rs b/app/buck2_audit_server/src/dep_files.rs index 25a440026d320..1979a6e29bd6b 100644 --- a/app/buck2_audit_server/src/dep_files.rs +++ b/app/buck2_audit_server/src/dep_files.rs @@ -18,8 +18,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use crate::AuditSubcommand; @@ -33,8 +33,9 @@ impl AuditSubcommand for AuditDepFilesCommand { ) -> anyhow::Result<()> { server_ctx .with_dice_ctx(async move |server_ctx, mut ctx| { - let target_platform = - target_platform_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(&client_ctx, server_ctx, &mut ctx) + .await?; let label = parse_patterns_from_cli_args::( &mut ctx, @@ -50,7 +51,7 @@ impl AuditSubcommand for AuditDepFilesCommand { .as_target_label(&self.pattern)?; let label = ctx - .get_configured_target(&label, target_platform.as_ref()) + .get_configured_target(&label, &global_cfg_options) .await?; let category = Category::try_from(self.category.as_str())?; diff --git a/app/buck2_audit_server/src/execution_platform_resolution.rs b/app/buck2_audit_server/src/execution_platform_resolution.rs index c9d068e347e47..4669ddc9eb4ba 100644 --- a/app/buck2_audit_server/src/execution_platform_resolution.rs +++ b/app/buck2_audit_server/src/execution_platform_resolution.rs @@ -25,7 +25,7 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::PatternParser; use indent_write::io::IndentWriter; @@ -78,7 +78,7 @@ impl AuditSubcommand for AuditExecutionPlatformResolutionCommand { } let loaded_patterns = load_patterns(&ctx, target_patterns, MissingTargetBehavior::Fail).await?; - let target_platform = target_platform_from_client_context( + let global_cfg_options = global_cfg_options_from_client_context( &client_ctx, server_ctx, &mut ctx, @@ -88,7 +88,7 @@ impl AuditSubcommand for AuditExecutionPlatformResolutionCommand { for (_, targets) in loaded_patterns.into_iter() { for (_, node) in targets? { configured_patterns.push( - ctx.get_configured_target(node.label(), target_platform.as_ref()) + ctx.get_configured_target(node.label(), &global_cfg_options) .await?, ); } diff --git a/app/buck2_audit_server/src/includes.rs b/app/buck2_audit_server/src/includes.rs index 90a751361cc17..3592fd7cbcc52 100644 --- a/app/buck2_audit_server/src/includes.rs +++ b/app/buck2_audit_server/src/includes.rs @@ -27,11 +27,11 @@ use buck2_interpreter::load_module::InterpreterCalculation; use buck2_interpreter::paths::module::StarlarkModulePath; use buck2_node::nodes::eval_result::EvaluationResult; use buck2_node::nodes::frontend::TargetGraphCalculation; -use buck2_query::query::environment::LabeledNode; -use buck2_query::query::environment::NodeLabel; +use buck2_query::query::graph::node::LabeledNode; +use buck2_query::query::graph::node::NodeKey; +use buck2_query::query::graph::successors::AsyncChildVisitor; use buck2_query::query::traversal::async_depth_first_postorder_traversal; use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::AsyncTraversalDelegate; use buck2_query::query::traversal::ChildVisitor; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; @@ -80,12 +80,12 @@ async fn get_transitive_includes( #[repr(transparent)] struct NodeRef(ImportPath); - impl NodeLabel for NodeRef {} + impl NodeKey for NodeRef {} impl LabeledNode for Node { - type NodeRef = NodeRef; + type Key = NodeRef; - fn node_ref(&self) -> &NodeRef { + fn node_key(&self) -> &NodeRef { NodeRef::ref_cast(self.import_path()) } } @@ -105,39 +105,37 @@ async fn get_transitive_includes( } } - struct Delegate { - imports: Vec, - } + let mut imports: Vec = Vec::new(); + struct Delegate; - #[async_trait] - impl AsyncTraversalDelegate for Delegate { - fn visit(&mut self, target: Node) -> anyhow::Result<()> { - self.imports.push(target.import_path().clone()); - Ok(()) - } + let visit = |target: Node| { + imports.push(target.import_path().clone()); + Ok(()) + }; + impl AsyncChildVisitor for Delegate { async fn for_each_child( - &mut self, + &self, target: &Node, - func: &mut dyn ChildVisitor, + mut func: impl ChildVisitor, ) -> anyhow::Result<()> { for import in target.0.imports() { - func.visit(NodeRef(import.clone()))?; + func.visit(&NodeRef(import.clone()))?; } Ok(()) } } - let mut delegate = Delegate { imports: vec![] }; let lookup = Lookup { ctx }; async_depth_first_postorder_traversal( &lookup, load_result.imports().map(NodeRef::ref_cast), - &mut delegate, + Delegate, + visit, ) .await?; - Ok(delegate.imports) + Ok(imports) } async fn load_and_collect_includes( @@ -160,8 +158,8 @@ async fn load_and_collect_includes( return Err(anyhow::anyhow!(AuditIncludesError::WrongBuildfilePath( path.clone(), buildfile_name.to_owned(), - ))) - .map_err(buck2_error::Error::from); + )) + .into()); } Ok(get_transitive_includes(ctx, &load_result).await?) diff --git a/app/buck2_audit_server/src/output/command.rs b/app/buck2_audit_server/src/output/command.rs index 663a594cc6846..78c4a2256f6df 100644 --- a/app/buck2_audit_server/src/output/command.rs +++ b/app/buck2_audit_server/src/output/command.rs @@ -18,14 +18,14 @@ use buck2_build_api::audit_output::AuditOutputResult; use buck2_build_api::audit_output::AUDIT_OUTPUT; use buck2_cli_proto::ClientContext; use buck2_common::dice::cells::HasCellResolver; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::CellResolver; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use dice::DiceComputations; use crate::output::buck_out_path_parser::BuckOutPathParser; @@ -45,7 +45,7 @@ async fn audit_output<'v>( working_dir: &'v ProjectRelativePath, cell_resolver: &'v CellResolver, dice_ctx: &'v DiceComputations, - global_target_platform: Option, + global_cfg_options: &'v GlobalCfgOptions, ) -> anyhow::Result> { let buck_out_parser = BuckOutPathParser::new(cell_resolver); let parsed = buck_out_parser.parse(output_path)?; @@ -69,7 +69,7 @@ async fn audit_output<'v>( }; let configured_target_label = dice_ctx - .get_configured_target(&target_label, global_target_platform.as_ref()) + .get_configured_target(&target_label, global_cfg_options) .await?; let command_config = configured_target_label.cfg(); @@ -86,7 +86,7 @@ async fn audit_output<'v>( Ok(FIND_MATCHING_ACTION.get()?( dice_ctx, working_dir, - global_target_platform, + global_cfg_options, &analysis, path_after_target_name, ) @@ -96,13 +96,13 @@ async fn audit_output<'v>( pub(crate) fn init_audit_output() { AUDIT_OUTPUT.init( - |output_path, working_dir, cell_resolver, dice_ctx, global_target_platform| { + |output_path, working_dir, cell_resolver, dice_ctx, global_cfg_options| { Box::pin(audit_output( output_path, working_dir, cell_resolver, dice_ctx, - global_target_platform, + global_cfg_options, )) }, ); @@ -128,14 +128,14 @@ impl AuditSubcommand for AuditOutputCommand { let working_dir = server_ctx.working_dir(); let cell_resolver = dice_ctx.get_cell_resolver().await?; - let global_target_platform = target_platform_from_client_context( + let global_cfg_options = global_cfg_options_from_client_context( &client_ctx, server_ctx, &mut dice_ctx, ) .await?; - let result = audit_output(&self.output_path, working_dir, &cell_resolver, &dice_ctx, global_target_platform).await?; + let result = audit_output(&self.output_path, working_dir, &cell_resolver, &dice_ctx, &global_cfg_options).await?; let mut stdout = stdout.as_writer(); diff --git a/app/buck2_audit_server/src/providers.rs b/app/buck2_audit_server/src/providers.rs index abef489a85372..8a67dac6ba3c6 100644 --- a/app/buck2_audit_server/src/providers.rs +++ b/app/buck2_audit_server/src/providers.rs @@ -24,8 +24,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_util::indent::indent; use dice::DiceTransaction; use dupe::Dupe; @@ -66,8 +66,8 @@ async fn server_execute_with_dice( mut ctx: DiceTransaction, ) -> anyhow::Result<()> { let cells = ctx.get_cell_resolver().await?; - let target_platform = - target_platform_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; let parsed_patterns = parse_patterns_from_cli_args::( &mut ctx, @@ -105,7 +105,7 @@ async fn server_execute_with_dice( for (target_name, providers) in targets { let label = providers.into_providers_label(package.dupe(), target_name.as_ref()); let providers_label = ctx - .get_configured_provider_label(&label, target_platform.as_ref()) + .get_configured_provider_label(&label, &global_cfg_options) .await?; // `.push` is deprecated in newer `futures`, diff --git a/app/buck2_audit_server/src/subtargets.rs b/app/buck2_audit_server/src/subtargets.rs index a46e95eef7a2c..8ce36bea9e555 100644 --- a/app/buck2_audit_server/src/subtargets.rs +++ b/app/buck2_audit_server/src/subtargets.rs @@ -27,8 +27,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::stdout_partial_output::StdoutPartialOutput; use buck2_util::indent::indent; use dice::DiceTransaction; @@ -64,8 +64,8 @@ async fn server_execute_with_dice( ) -> anyhow::Result<()> { // TODO(raulgarcia4): Extract function where possible, shares a lot of code with audit providers. let cells = ctx.get_cell_resolver().await?; - let target_platform = - target_platform_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(&client_ctx, server_ctx, &mut ctx).await?; let parsed_patterns = parse_patterns_from_cli_args::( &mut ctx, @@ -104,7 +104,7 @@ async fn server_execute_with_dice( for (target_name, providers) in targets { let label = providers.into_providers_label(package.dupe(), target_name.as_ref()); let providers_label = ctx - .get_configured_provider_label(&label, target_platform.as_ref()) + .get_configured_provider_label(&label, &global_cfg_options) .await?; // `.push` is deprecated in newer `futures`, diff --git a/app/buck2_audit_server/src/visibility.rs b/app/buck2_audit_server/src/visibility.rs index e8bbf9c2e2b2f..78fd605074a11 100644 --- a/app/buck2_audit_server/src/visibility.rs +++ b/app/buck2_audit_server/src/visibility.rs @@ -16,10 +16,9 @@ use buck2_node::load_patterns::MissingTargetBehavior; use buck2_node::nodes::lookup::TargetNodeLookup; use buck2_node::nodes::unconfigured::TargetNode; use buck2_node::visibility::VisibilityError; +use buck2_query::query::environment::QueryTargetDepsSuccessors; use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::traversal::async_depth_first_postorder_traversal; -use buck2_query::query::traversal::AsyncTraversalDelegate; -use buck2_query::query::traversal::ChildVisitor; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::ctx::ServerCommandDiceContext; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; @@ -42,41 +41,28 @@ async fn verify_visibility( ctx: DiceTransaction, targets: TargetSet, ) -> anyhow::Result<()> { - struct Delegate { - targets: TargetSet, - } + let mut new_targets: TargetSet = TargetSet::new(); - #[async_trait] - impl AsyncTraversalDelegate for Delegate { - fn visit(&mut self, target: TargetNode) -> anyhow::Result<()> { - self.targets.insert(target); - Ok(()) - } - async fn for_each_child( - &mut self, - target: &TargetNode, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - for dep in target.deps() { - func.visit(dep.dupe())?; - } - Ok(()) - } - } + let visit = |target| { + new_targets.insert(target); + Ok(()) + }; let lookup = TargetNodeLookup(&ctx); - let mut delegate = Delegate { - targets: TargetSet::::new(), - }; - - async_depth_first_postorder_traversal(&lookup, targets.iter_names(), &mut delegate).await?; + async_depth_first_postorder_traversal( + &lookup, + targets.iter_names(), + QueryTargetDepsSuccessors, + visit, + ) + .await?; let mut visibility_errors = Vec::new(); - for target in delegate.targets.iter() { + for target in new_targets.iter() { for dep in target.deps() { - match delegate.targets.get(dep) { + match new_targets.get(dep) { Some(val) => { if !val.is_visible_to(target.label())? { visibility_errors.push(VisibilityError::NotVisibleTo( @@ -134,7 +120,7 @@ impl AuditSubcommand for AuditVisibilityCommand { let mut nodes = TargetSet::::new(); for (_package, result) in parsed_target_patterns.iter() { let res = result.as_ref().map_err(Dupe::dupe)?; - nodes.extend(res.values()); + nodes.extend(res.values().map(|n| n.to_owned())); } verify_visibility(ctx, nodes).await?; diff --git a/app/buck2_build_api/BUCK b/app/buck2_build_api/BUCK index d5327bdc4b100..d665ca3026075 100644 --- a/app/buck2_build_api/BUCK +++ b/app/buck2_build_api/BUCK @@ -30,7 +30,6 @@ rust_library( "fbsource//third-party/rust:regex", "fbsource//third-party/rust:serde", "fbsource//third-party/rust:serde_json", - "fbsource//third-party/rust:shlex", "fbsource//third-party/rust:smallvec", "fbsource//third-party/rust:static_assertions", "fbsource//third-party/rust:tokio", @@ -56,6 +55,7 @@ rust_library( "//buck2/app/buck2_query:buck2_query", "//buck2/app/buck2_test_api:buck2_test_api", "//buck2/app/buck2_util:buck2_util", + "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/dice/dice:dice", "//buck2/gazebo/display_container:display_container", "//buck2/gazebo/dupe:dupe", diff --git a/app/buck2_build_api/Cargo.toml b/app/buck2_build_api/Cargo.toml index f6e86f5b53960..d81a4aa62bce3 100644 --- a/app/buck2_build_api/Cargo.toml +++ b/app/buck2_build_api/Cargo.toml @@ -27,7 +27,6 @@ ref-cast = { workspace = true } regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -shlex = { workspace = true } smallvec = { workspace = true } static_assertions = { workspace = true } tokio = { workspace = true } @@ -64,6 +63,7 @@ buck2_node = { workspace = true } buck2_query = { workspace = true } buck2_test_api = { workspace = true } buck2_util = { workspace = true } +buck2_wrapper_common = { workspace = true } [dev-dependencies] buck2_wrapper_common = { workspace = true } diff --git a/app/buck2_build_api/src/actions/artifact/materializer.rs b/app/buck2_build_api/src/actions/artifact/materializer.rs index b0e7007f04b59..fc8981d509669 100644 --- a/app/buck2_build_api/src/actions/artifact/materializer.rs +++ b/app/buck2_build_api/src/actions/artifact/materializer.rs @@ -80,6 +80,7 @@ impl ArtifactMaterializer for DiceComputations { NodeDuration { user: duration, total: duration, + queue: None, }, current_span(), ); diff --git a/app/buck2_build_api/src/actions/calculation.rs b/app/buck2_build_api/src/actions/calculation.rs index 9bac10f16dfe7..f0c7558d301e3 100644 --- a/app/buck2_build_api/src/actions/calculation.rs +++ b/app/buck2_build_api/src/actions/calculation.rs @@ -97,9 +97,17 @@ async fn build_action_no_redirect( ) -> anyhow::Result { let materialized_inputs = { let inputs = action.inputs()?; + let ensure_futs: FuturesOrdered<_> = inputs .iter() - .map(|v| ensure_artifact_group_staged(ctx, v)) + .map(|v| async { + let resolved = v.resolved_artifact(ctx).await?; + anyhow::Ok( + ensure_artifact_group_staged(ctx, resolved.clone()) + .await? + .to_group_values(&resolved)?, + ) + }) .collect(); let ready_inputs: Vec<_> = @@ -107,7 +115,7 @@ async fn build_action_no_redirect( let mut results = IndexMap::with_capacity(inputs.len()); for (artifact, ready) in zip(inputs.iter(), ready_inputs) { - results.insert(artifact.clone(), ready.to_group_values(artifact)?); + results.insert(artifact.clone(), ready); } results }; @@ -143,6 +151,8 @@ async fn build_action_no_redirect( ) .await; + let queue_duration = command_reports.last().and_then(|r| r.timing.queue_duration); + let action_key = action.key().as_proto(); let action_name = buck2_data::ActionName { @@ -251,7 +261,7 @@ async fn build_action_no_redirect( .unwrap_or_default(); ( - (action_result, wall_time), + (action_result, wall_time, queue_duration), Box::new(buck2_data::ActionExecutionEnd { key: Some(action_key), kind: action.kind().into(), @@ -282,7 +292,7 @@ async fn build_action_no_redirect( }; // boxed() the future so that we don't need to allocate space for it while waiting on input dependencies. - let ((res, wall_time), spans) = + let ((res, wall_time, queue_duration), spans) = async_record_root_spans(span_async(start_event, fut.boxed())).await; // TODO: This wall time is rather wrong. We should report a wall time on failures too. @@ -291,6 +301,7 @@ async fn build_action_no_redirect( duration: NodeDuration { user: wall_time.unwrap_or_default(), total: now.elapsed(), + queue: queue_duration, }, spans, })?; diff --git a/app/buck2_build_api/src/actions/error_handler.rs b/app/buck2_build_api/src/actions/error_handler.rs index 60e70ff8cf820..3de4bd031fb60 100644 --- a/app/buck2_build_api/src/actions/error_handler.rs +++ b/app/buck2_build_api/src/actions/error_handler.rs @@ -16,6 +16,7 @@ use buck2_data::ActionSubError; use buck2_data::CommandExecution; use derive_more::Display; use display_container::fmt_container; +use gazebo::prelude::SliceClonedExt; use starlark::environment::GlobalsBuilder; use starlark::environment::Methods; use starlark::environment::MethodsBuilder; @@ -25,6 +26,7 @@ use starlark::starlark_simple_value; use starlark::typing::Ty; use starlark::values::list::UnpackList; use starlark::values::list_or_tuple::UnpackListOrTuple; +use starlark::values::none::NoneOr; use starlark::values::starlark_value; use starlark::values::starlark_value_as_type::StarlarkValueAsType; use starlark::values::AllocValue; @@ -34,8 +36,11 @@ use starlark::values::ProvidesStaticType; use starlark::values::StarlarkValue; use starlark::values::Trace; use starlark::values::Value; +use starlark::values::ValueError; use starlark::StarlarkDocs; +use crate::starlark::values::ValueLike; + pub(crate) type ActionSubErrorResult<'a> = UnpackList<&'a StarlarkActionSubError<'a>>; #[derive(Debug, buck2_error::Error)] @@ -113,10 +118,13 @@ fn action_error_context_methods(builder: &mut MethodsBuilder) { fn new_error_location<'v>( #[starlark(this)] _this: &'v StarlarkActionErrorContext, #[starlark(require = named)] file: String, - #[starlark(require = named)] line: Option, + #[starlark(require = named, default = NoneOr::None)] line: NoneOr, ) -> anyhow::Result { // @TODO(wendyy) - actually enforce/validate the path types. - Ok(StarlarkActionErrorLocation { file, line }) + Ok(StarlarkActionErrorLocation { + file, + line: line.into_option(), + }) } /// Create a new sub error, specifying an error category name, optional message, and @@ -133,15 +141,15 @@ fn action_error_context_methods(builder: &mut MethodsBuilder) { fn new_sub_error<'v>( #[starlark(this)] _this: &'v StarlarkActionErrorContext, #[starlark(require = named)] category: String, - #[starlark(require = named)] message: Option, - #[starlark(require = named)] locations: Option< + #[starlark(require = named, default = NoneOr::None)] message: NoneOr, + #[starlark(require = named, default = NoneOr::None)] locations: NoneOr< UnpackListOrTuple<&'v StarlarkActionErrorLocation>, >, ) -> anyhow::Result> { Ok(StarlarkActionSubError { category, - message, - locations, + message: message.into_option(), + locations: locations.into_option(), }) } } @@ -155,7 +163,11 @@ fn action_error_context_methods(builder: &mut MethodsBuilder) { Display, NoSerialize, Clone, - Default + Default, + Ord, + PartialOrd, + Eq, + PartialEq )] #[display( fmt = "ActionErrorLocation(file={}, line={})", @@ -168,7 +180,28 @@ pub struct StarlarkActionErrorLocation { } #[starlark_value(type = "ActionErrorLocation", StarlarkTypeRepr, UnpackValue)] -impl<'v> StarlarkValue<'v> for StarlarkActionErrorLocation {} +impl<'v> StarlarkValue<'v> for StarlarkActionErrorLocation { + fn get_methods() -> Option<&'static Methods> { + static RES: MethodsStatic = MethodsStatic::new(); + RES.methods(action_error_location_methods) + } + + fn equals(&self, other: Value<'v>) -> starlark::Result { + if let Some(other) = other.downcast_ref::() { + Ok(self.eq(other)) + } else { + Ok(false) + } + } + + fn compare(&self, other: Value<'v>) -> starlark::Result { + if let Some(other) = other.downcast_ref::() { + Ok(self.cmp(other)) + } else { + ValueError::unsupported_with(self, "compare", other) + } + } +} impl<'v> AllocValue<'v> for StarlarkActionErrorLocation { fn alloc_value(self, heap: &'v Heap) -> Value<'v> { @@ -176,6 +209,25 @@ impl<'v> AllocValue<'v> for StarlarkActionErrorLocation { } } +/// Methods available on `StarlarkActionErrorLocation` to help with testing the error +/// handler implementation +#[starlark_module] +fn action_error_location_methods(builder: &mut MethodsBuilder) { + /// The file of the error location. This is only needed for action error handler + /// unit testing. + #[starlark(attribute)] + fn file<'v>(this: &'v StarlarkActionErrorLocation) -> anyhow::Result<&'v str> { + Ok(&this.file) + } + + /// The line of the error location. This is only needed for action error handler + /// unit testing. + #[starlark(attribute)] + fn line<'v>(this: &'v StarlarkActionErrorLocation) -> anyhow::Result> { + Ok(this.line) + } +} + #[derive( ProvidesStaticType, Trace, @@ -183,7 +235,11 @@ impl<'v> AllocValue<'v> for StarlarkActionErrorLocation { StarlarkDocs, Debug, NoSerialize, - Clone + Clone, + Ord, + PartialOrd, + Eq, + PartialEq )] pub(crate) struct StarlarkActionSubError<'v> { category: String, @@ -218,7 +274,59 @@ impl<'v> AllocValue<'v> for StarlarkActionSubError<'v> { } #[starlark_value(type = "ActionSubError", StarlarkTypeRepr, UnpackValue)] -impl<'v> StarlarkValue<'v> for StarlarkActionSubError<'v> {} +impl<'v> StarlarkValue<'v> for StarlarkActionSubError<'v> { + fn get_methods() -> Option<&'static Methods> { + static RES: MethodsStatic = MethodsStatic::new(); + RES.methods(action_sub_error_methods) + } + + fn equals(&self, other: Value<'v>) -> starlark::Result { + if let Some(other) = other.downcast_ref::() { + Ok(self.eq(other)) + } else { + Ok(false) + } + } + + fn compare(&self, other: Value<'v>) -> starlark::Result { + if let Some(other) = other.downcast_ref::() { + Ok(self.cmp(other)) + } else { + ValueError::unsupported_with(self, "compare", other) + } + } +} + +/// Methods available on `StarlarkActionSubError` to help with testing the error +/// handler implementation +#[starlark_module] +fn action_sub_error_methods(builder: &mut MethodsBuilder) { + /// The category name of this sub error. This function is only needed for action + /// error handler unit testing. + #[starlark(attribute)] + fn category<'v>(this: &'v StarlarkActionSubError) -> anyhow::Result<&'v str> { + Ok(&this.category) + } + + /// The optional message associated with this sub error. This function is only + /// needed for action error handler unit testing. + #[starlark(attribute)] + fn message<'v>(this: &'v StarlarkActionSubError) -> anyhow::Result> { + Ok(this.message.as_deref()) + } + + /// Any locations associated with this sub error. This function is only needed + /// for action error handler unit testing. + #[starlark(attribute)] + fn locations<'v>( + this: &'v StarlarkActionSubError, + ) -> anyhow::Result>> { + Ok(this + .locations + .as_ref() + .map(|locations| locations.items.cloned())) + } +} impl<'v> StarlarkActionSubError<'v> { pub(crate) fn to_proto(&self) -> ActionSubError { @@ -250,3 +358,19 @@ pub(crate) fn register_action_error_types(globals: &mut GlobalsBuilder) { const ActionErrorLocation: StarlarkValueAsType = StarlarkValueAsType::new(); } + +/// Global methods for testing starlark action error handler. +#[starlark_module] +pub(crate) fn register_action_error_handler_for_testing(builder: &mut GlobalsBuilder) { + /// Global function to create a new `ActionErrorContext` for testing a starlark action error + /// handler via `bxl_test`. + fn new_test_action_error_ctx( + #[starlark(require=named, default = "")] stderr: &str, + #[starlark(require=named, default = "")] stdout: &str, + ) -> anyhow::Result { + Ok(StarlarkActionErrorContext { + stderr: stderr.to_owned(), + stdout: stdout.to_owned(), + }) + } +} diff --git a/app/buck2_build_api/src/actions/execute/action_executor.rs b/app/buck2_build_api/src/actions/execute/action_executor.rs index 6c6d8c9ffc209..87ae7d1740f1b 100644 --- a/app/buck2_build_api/src/actions/execute/action_executor.rs +++ b/app/buck2_build_api/src/actions/execute/action_executor.rs @@ -359,7 +359,7 @@ impl ActionExecutionCtx for BuckActionExecutionContext<'_> { } fn artifact_values(&self, artifact: &ArtifactGroup) -> &ArtifactGroupValues { - self.inputs.get(artifact).unwrap_or_else(|| panic!("Internal error: action {:?} tried to grab the artifact {:?} even though it was not an input.", self.action.owner(), artifact)) + self.inputs.get(artifact).unwrap_or_else(|| panic!("Internal error: action {} tried to grab the artifact {} even though it was not an input.", self.action.owner(), artifact)) } fn blocking_executor(&self) -> &dyn BlockingExecutor { diff --git a/app/buck2_build_api/src/actions/impls/expanded_command_line.rs b/app/buck2_build_api/src/actions/impls/expanded_command_line.rs index 004738abeb77d..12bc8100cc892 100644 --- a/app/buck2_build_api/src/actions/impls/expanded_command_line.rs +++ b/app/buck2_build_api/src/actions/impls/expanded_command_line.rs @@ -67,7 +67,7 @@ impl ExpandedCommandLine { } #[cfg(test)] -mod test { +mod tests { use sorted_vector_map::sorted_vector_map; use sorted_vector_map::SortedVectorMap; diff --git a/app/buck2_build_api/src/actions/mod.rs b/app/buck2_build_api/src/actions/mod.rs index 202f4a2fd7efc..06c1ddbefadae 100644 --- a/app/buck2_build_api/src/actions/mod.rs +++ b/app/buck2_build_api/src/actions/mod.rs @@ -274,6 +274,7 @@ pub trait ActionExecutionCtx: Send + Sync { } #[derive(buck2_error::Error, Debug)] +#[buck2(user)] pub enum ActionErrors { #[error("Output path for artifact or metadata file cannot be empty.")] EmptyOutputPath, diff --git a/app/buck2_build_api/src/actions/query.rs b/app/buck2_build_api/src/actions/query.rs index 17d7b7a3f6b98..3b08fed227100 100644 --- a/app/buck2_build_api/src/actions/query.rs +++ b/app/buck2_build_api/src/actions/query.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use allocative::Allocative; use buck2_artifact::actions::key::ActionKey; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::cells::CellResolver; @@ -25,21 +26,20 @@ use buck2_core::fs::artifact_path_resolver::ArtifactFs; use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; use buck2_core::fs::project_rel_path::ProjectRelativePath; use buck2_core::provider::label::ConfiguredProvidersLabel; -use buck2_core::target::label::TargetLabel; use buck2_execute::artifact::fs::ExecutorFs; -use buck2_query::query::environment::LabeledNode; -use buck2_query::query::environment::NodeLabel; use buck2_query::query::environment::QueryTarget; +use buck2_query::query::graph::node::LabeledNode; +use buck2_query::query::graph::node::NodeKey; use buck2_util::late_binding::LateBinding; use derivative::Derivative; use dice::DiceComputations; use dupe::Dupe; +use either::Either; use gazebo::variants::VariantName; use indexmap::IndexMap; use internment::ArcIntern; use ref_cast::RefCast; use serde::Serialize; -use serde::Serializer; use crate::actions::RegisteredAction; use crate::analysis::AnalysisResult; @@ -180,9 +180,9 @@ impl ActionQueryNode { } impl LabeledNode for ActionQueryNode { - type NodeRef = ActionQueryNodeRef; + type Key = ActionQueryNodeRef; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { &self.key } } @@ -247,7 +247,7 @@ pub enum ActionQueryNodeRef { Action(ActionKey), } -impl NodeLabel for ActionQueryNodeRef {} +impl NodeKey for ActionQueryNodeRef {} impl ActionQueryNodeRef { pub fn require_action(&self) -> anyhow::Result<&ActionKey> { @@ -276,23 +276,22 @@ impl QueryTarget for ActionQueryNode { unimplemented!("buildfile not yet implemented in aquery") } - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn deps<'a>(&'a self) -> Box + Send + 'a> { + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a { // When traversing deps in aquery, we do *not* traverse deps for the target nodes, since // those are just for literals let action = match &self.data { ActionQueryNodeData::Action(action) => action, - ActionQueryNodeData::Analysis(..) => return Box::new(std::iter::empty()), + ActionQueryNodeData::Analysis(..) => return Either::Left(std::iter::empty()), }; - Box::new(iter_action_inputs(&action.deps)) + Either::Right(iter_action_inputs(&action.deps)) } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(std::iter::empty()) + fn exec_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + std::iter::empty() } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { + fn target_deps<'a>(&'a self) -> impl Iterator + Send + 'a { self.deps() } @@ -361,22 +360,6 @@ impl QueryTarget for ActionQueryNode { // TODO(cjhopman): In addition to implementing this, we should be able to return an anyhow::Error here rather than panicking. unimplemented!("inputs not yet implemented in aquery") } - - fn call_stack(&self) -> Option { - None - } - - fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { - format!("{:#}", attr) - } - - fn attr_serialize( - &self, - attr: &Self::Attr<'_>, - serializer: S, - ) -> Result { - attr.serialize(serializer) - } } pub fn iter_action_inputs<'a>( @@ -436,8 +419,8 @@ pub static FIND_MATCHING_ACTION: LateBinding< &'c DiceComputations, // Working dir. &'c ProjectRelativePath, - // global_target_platform - Option, + // target cfg info (target platform, cli modifiers) + &'c GlobalCfgOptions, &'c AnalysisResult, // path_after_target_name ForwardRelativePathBuf, diff --git a/app/buck2_build_api/src/artifact_groups/calculation.rs b/app/buck2_build_api/src/artifact_groups/calculation.rs index 3790a352b0a25..447128d0a995d 100644 --- a/app/buck2_build_api/src/artifact_groups/calculation.rs +++ b/app/buck2_build_api/src/artifact_groups/calculation.rs @@ -74,9 +74,10 @@ impl ArtifactGroupCalculation for DiceComputations { input: &ArtifactGroup, ) -> anyhow::Result { // TODO consider if we need to cache this - ensure_artifact_group_staged(self, input) + let resolved_artifacts = input.resolved_artifact(self).await?; + ensure_artifact_group_staged(self, resolved_artifacts.clone()) .await? - .to_group_values(input) + .to_group_values(&resolved_artifacts) } } @@ -100,9 +101,9 @@ impl ArtifactGroupCalculation for DiceComputations { /// inputs are ready. pub(crate) fn ensure_artifact_group_staged<'a>( ctx: &'a DiceComputations, - input: &'a ArtifactGroup, + input: ResolvedArtifactGroup<'a>, ) -> impl Future> + 'a { - match input.assert_resolved() { + match input { ResolvedArtifactGroup::Artifact(artifact) => { ensure_artifact_staged(ctx, artifact.clone()).left_future() } @@ -119,11 +120,9 @@ pub(super) fn ensure_base_artifact_staged<'a>( artifact: BaseArtifactKind, ) -> impl Future> + 'a { match artifact { - BaseArtifactKind::Build(built) => { - ensure_build_artifact_staged(dice, built.clone()).left_future() - } + BaseArtifactKind::Build(built) => ensure_build_artifact_staged(dice, built).left_future(), BaseArtifactKind::Source(source) => { - ensure_source_artifact_staged(dice, source.clone()).right_future() + ensure_source_artifact_staged(dice, source).right_future() } } } @@ -146,11 +145,9 @@ fn ensure_build_artifact_staged<'a>( dice: &'a DiceComputations, built: BuildArtifact, ) -> impl Future> + 'a { - let key = built.key().clone(); - let path = built.get_path().clone(); - ActionCalculation::build_action(dice, key.clone()).map(move |action_outputs| { + ActionCalculation::build_action(dice, built.key.clone()).map(move |action_outputs| { let action_outputs = action_outputs?; - if let Some(value) = action_outputs.get(&path) { + if let Some(value) = action_outputs.get(&built.path) { Ok(EnsureArtifactGroupReady::Single(value.dupe())) } else { Err( @@ -198,13 +195,13 @@ pub(crate) enum EnsureArtifactGroupReady { impl EnsureArtifactGroupReady { /// Converts the ensured artifact to an ArtifactGroupValues. The caller must ensure that the passed in artifact /// is the same one that was used to ensure this. - pub(crate) fn to_group_values( + pub(crate) fn to_group_values<'v>( self, - artifact: &ArtifactGroup, + resolved_artifact_group: &ResolvedArtifactGroup<'v>, ) -> anyhow::Result { match self { EnsureArtifactGroupReady::TransitiveSet(values) => Ok(values), - EnsureArtifactGroupReady::Single(value) => match artifact.assert_resolved() { + EnsureArtifactGroupReady::Single(value) => match resolved_artifact_group { ResolvedArtifactGroup::Artifact(artifact) => { Ok(ArtifactGroupValues::from_artifact(artifact.clone(), value)) } @@ -231,9 +228,38 @@ static_assertions::assert_eq_size!(EnsureArtifactGroupReady, [usize; 3]); // TODO(cjhopman): We should be able to wrap this in a convenient assertion macro. #[allow(unused, clippy::diverging_sub_expression)] fn _assert_ensure_artifact_group_future_size() { - let v = ensure_artifact_group_staged(panic!(), panic!()); + let ctx: DiceComputations = panic!(); + + // These first two are the important ones to track and not regress. + let v = ctx.ensure_artifact_group(panic!()); + let e = [0u8; 128 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); + + let v = ensure_artifact_group_staged(&ctx, panic!()); let e = [0u8; 704 / 8]; static_assertions::assert_eq_size_ptr!(&v, &e); + + // The rest of these are to help understand how changes are impacting the important ones above. Regressing these + // is generally okay if the above don't regress. + let v = ensure_artifact_staged(&ctx, panic!()); + let e = [0u8; 640 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); + + let v = ensure_base_artifact_staged(&ctx, panic!()); + let e = [0u8; 512 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); + + let v = ensure_build_artifact_staged(&ctx, panic!()); + let e = [0u8; 512 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); + + let v = ActionCalculation::build_action(&ctx, panic!()); + let e = [0u8; 128 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); + + let v = ensure_source_artifact_staged(&ctx, panic!()); + let e = [0u8; 128 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); } async fn dir_artifact_value( @@ -394,9 +420,16 @@ impl Key for EnsureTransitiveSetProjectionKey { let artifact_fs = ctx.get_artifact_fs().await?; - let sub_inputs = set + let projection_sub_inputs = set .as_transitive_set() .get_projection_sub_inputs(self.0.projection)?; + let sub_inputs_futs: FuturesOrdered<_> = projection_sub_inputs + .iter() + .map(|a| async { a.resolved_artifact(ctx).await }) + .collect(); + + let sub_inputs: Vec<_> = + tokio::task::unconstrained(keep_going::try_join_all(ctx, sub_inputs_futs)).await?; let (values, children) = { // Compute the new inputs. Note that ordering here (and below) is important to ensure @@ -405,7 +438,9 @@ impl Key for EnsureTransitiveSetProjectionKey { let ensure_futs: FuturesOrdered<_> = sub_inputs .iter() - .map(|v| ensure_artifact_group_staged(ctx, v)) + .map(|v: &ResolvedArtifactGroup| async { + ensure_artifact_group_staged(ctx, v.clone()).await + }) .collect(); let ready_inputs: Vec<_> = @@ -414,7 +449,7 @@ impl Key for EnsureTransitiveSetProjectionKey { // Partition our inputs in artifacts and projections. let mut values_count = 0; for input in sub_inputs.iter() { - if let ArtifactGroup::Artifact(..) = input { + if let ResolvedArtifactGroup::Artifact(..) = input { values_count += 1; } } @@ -423,7 +458,7 @@ impl Key for EnsureTransitiveSetProjectionKey { let mut children = Vec::with_capacity(sub_inputs.len() - values_count); for (group, ready) in zip(sub_inputs.iter(), ready_inputs) { - match group.assert_resolved() { + match group { ResolvedArtifactGroup::Artifact(artifact) => { values.push((artifact.dupe(), ready.unpack_single()?)) } diff --git a/app/buck2_build_api/src/artifact_groups/mod.rs b/app/buck2_build_api/src/artifact_groups/mod.rs index e69a2dd4fbcf4..16eb9467f1a07 100644 --- a/app/buck2_build_api/src/artifact_groups/mod.rs +++ b/app/buck2_build_api/src/artifact_groups/mod.rs @@ -12,7 +12,8 @@ pub mod calculation; pub mod deferred; pub mod promise; -use crate::interpreter::rule_defs::context::get_artifact_from_anon_target_analysis; +use crate::actions::calculation::BuildKey; +use crate::deferred::calculation::GET_PROMISED_ARTIFACT; pub mod registry; @@ -26,6 +27,7 @@ use dice::DiceComputations; use dupe::Dupe; use gazebo::variants::UnpackVariants; +use self::calculation::EnsureTransitiveSetProjectionKey; use crate::artifact_groups::deferred::TransitiveSetKey; use crate::artifact_groups::promise::PromiseArtifact; @@ -49,22 +51,6 @@ pub enum ArtifactGroup { } impl ArtifactGroup { - // TODO(@wendyy) - deprecate - pub fn assert_resolved(&self) -> ResolvedArtifactGroup { - self.resolved().unwrap() - } - - // TODO(@wendyy) - deprecate - pub fn resolved(&self) -> anyhow::Result { - Ok(match self { - ArtifactGroup::Artifact(a) => ResolvedArtifactGroup::Artifact(a.clone()), - ArtifactGroup::TransitiveSetProjection(a) => { - ResolvedArtifactGroup::TransitiveSetProjection(a) - } - ArtifactGroup::Promise(p) => ResolvedArtifactGroup::Artifact(p.get_err()?.clone()), - }) - } - /// Gets the resolved artifact group, which is used further downstream to use DICE to get /// or compute the actual artifact values. For the `Artifact` variant, we will get the results /// via the base or projected artifact key. For the `TransitiveSetProjection` variant, we will @@ -83,7 +69,7 @@ impl ArtifactGroup { ArtifactGroup::Promise(p) => match p.get() { Some(a) => ResolvedArtifactGroup::Artifact(a.clone()), None => { - let artifact = get_artifact_from_anon_target_analysis(p, ctx).await?; + let artifact = (GET_PROMISED_ARTIFACT.get()?)(p, ctx).await?; ResolvedArtifactGroup::Artifact(artifact) } }, @@ -94,11 +80,17 @@ impl ArtifactGroup { // TODO(@wendyy) if we move PromiseArtifact into ArtifactKind someday, we should probably // split the Artifact variant into two cases (artifact by ref and by value) to prevent memory // regressions. +#[derive(Clone)] pub enum ResolvedArtifactGroup<'a> { Artifact(Artifact), TransitiveSetProjection(&'a TransitiveSetProjectionKey), } +pub enum ResolvedArtifactGroupBuildSignalsKey { + EnsureTransitiveSetProjectionKey(EnsureTransitiveSetProjectionKey), + BuildKey(BuildKey), +} + #[derive(Clone, Debug, Display, Dupe, PartialEq, Eq, Hash, Allocative)] #[display(fmt = "TransitiveSetProjection({}, {})", key, projection)] pub struct TransitiveSetProjectionKey { diff --git a/app/buck2_build_api/src/artifact_groups/promise.rs b/app/buck2_build_api/src/artifact_groups/promise.rs index 80f2f80cddf43..1bc97fba23933 100644 --- a/app/buck2_build_api/src/artifact_groups/promise.rs +++ b/app/buck2_build_api/src/artifact_groups/promise.rs @@ -21,11 +21,8 @@ use starlark::codemap::FileSpan; #[derive(Debug, buck2_error::Error)] pub enum PromiseArtifactResolveError { - #[error( - "Resolved promise of the artifact promise {} was not an artifact (was `{1}`)", - maybe_declared_at(_0) - )] - NotAnArtifact(Option, String), + #[error("Resolved promise of the artifact promise was not an artifact (was `{0}`)")] + NotAnArtifact(String), #[error("Artifact promise {1} {} wasn't resolved", maybe_declared_at(_0))] PromiseNotResolved(Option, String), #[error("Artifact promise was resolved multiple times")] @@ -44,10 +41,12 @@ pub enum PromiseArtifactResolveError { "assert_short_path() was called with `short_path = {0}`, but it did not match the artifact's actual short path: `{1}`" )] ShortPathMismatch(ForwardRelativePathBuf, String), - #[error("Internal error: analysis result did not contain promise ({0})")] - NotFoundInAnalysis(PromiseArtifact), - #[error("Internal error: promise artifact ({0}) owner is ({1}), which is not an anon target")] - OwnerIsNotAnonTarget(PromiseArtifact, BaseDeferredKey), + #[error("Internal error: analysis result did not contain promise with ID ({0})")] + NotFoundInAnalysis(PromiseArtifactId), + #[error( + "Internal error: promise artifact (id: {0}) owner is ({1}), which is not an anon target" + )] + OwnerIsNotAnonTarget(PromiseArtifactId, BaseDeferredKey), } fn maybe_declared_at(location: &Option) -> String { @@ -77,6 +76,16 @@ impl PromiseArtifactId { pub fn new(owner: BaseDeferredKey, id: usize) -> PromiseArtifactId { Self { owner, id } } + + pub fn owner(&self) -> &BaseDeferredKey { + &self.owner + } +} + +impl Display for PromiseArtifactId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}#{}", &self.owner, self.id) + } } impl PromiseArtifact { diff --git a/app/buck2_build_api/src/audit_output.rs b/app/buck2_build_api/src/audit_output.rs index 5c0d7cb303d1f..652ced85c0edd 100644 --- a/app/buck2_build_api/src/audit_output.rs +++ b/app/buck2_build_api/src/audit_output.rs @@ -10,6 +10,7 @@ use std::future::Future; use std::pin::Pin; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::CellResolver; use buck2_core::fs::project_rel_path::ProjectRelativePath; use buck2_core::target::label::TargetLabel; @@ -33,7 +34,7 @@ pub static AUDIT_OUTPUT: LateBinding< &'v ProjectRelativePath, &'v CellResolver, &'v DiceComputations, - global_target_platform: Option, + &'v GlobalCfgOptions, ) -> Pin>> + 'v>>, > = LateBinding::new("AUDIT_OUTPUT"); @@ -43,14 +44,14 @@ pub async fn audit_output<'v>( working_dir: &'v ProjectRelativePath, cell_resolver: &'v CellResolver, dice_ctx: &'v DiceComputations, - global_target_platform: Option, + global_cfg_options: &'v GlobalCfgOptions, ) -> anyhow::Result> { (AUDIT_OUTPUT.get()?)( output_path, working_dir, cell_resolver, dice_ctx, - global_target_platform, + global_cfg_options, ) .await } diff --git a/app/buck2_server_commands/src/commands/build/action_error.rs b/app/buck2_build_api/src/build/action_error.rs similarity index 99% rename from app/buck2_server_commands/src/commands/build/action_error.rs rename to app/buck2_build_api/src/build/action_error.rs index d38138059041f..2b2030dd91059 100644 --- a/app/buck2_server_commands/src/commands/build/action_error.rs +++ b/app/buck2_build_api/src/build/action_error.rs @@ -17,7 +17,7 @@ use buck2_event_observer::display::get_action_error_reason; use buck2_event_observer::display::TargetDisplayOptions; use serde::Serialize; -use crate::commands::build::BuildReportCollector; +use crate::build::build_report::BuildReportCollector; #[derive(Debug, Clone, Serialize, PartialOrd, Ord, PartialEq, Eq)] struct BuildReportActionName { diff --git a/app/buck2_server_commands/src/commands/build/build_report.rs b/app/buck2_build_api/src/build/build_report.rs similarity index 90% rename from app/buck2_server_commands/src/commands/build/build_report.rs rename to app/buck2_build_api/src/build/build_report.rs index 3f2411b3a0890..9caee1da1da33 100644 --- a/app/buck2_server_commands/src/commands/build/build_report.rs +++ b/app/buck2_build_api/src/build/build_report.rs @@ -17,9 +17,6 @@ use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; -use buck2_build_api::build::BuildProviderType; -use buck2_build_api::build::BuildTargetResult; -use buck2_build_api::build::ConfiguredBuildTargetResult; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::configuration::data::ConfigurationData; use buck2_core::fs::artifact_path_resolver::ArtifactFs; @@ -28,6 +25,7 @@ use buck2_core::fs::project::ProjectRoot; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::NonDefaultProvidersName; +use buck2_core::provider::label::ProvidersLabel; use buck2_core::provider::label::ProvidersName; use buck2_core::target::label::TargetLabel; use buck2_error::UniqueRootId; @@ -42,7 +40,9 @@ use itertools::Itertools; use serde::Serialize; use starlark_map::small_set::SmallSet; -use crate::commands::build::action_error::BuildReportActionError; +use crate::build::action_error::BuildReportActionError; +use crate::build::BuildProviderType; +use crate::build::ConfiguredBuildTargetResult; #[derive(Debug, Serialize)] #[allow(clippy::upper_case_acronyms)] // We care about how they serialise @@ -61,11 +61,12 @@ impl Default for BuildOutcome { /// DO NOT UPDATE WITHOUT UPDATING `docs/users/build_observability/build_report.md`! #[derive(Debug, Serialize)] -pub(crate) struct BuildReport { +pub struct BuildReport { trace_id: TraceId, success: bool, results: HashMap, - failures: HashMap, + /// filled only when fill-out-failures is passed for Buck1 backcompat only + failures: HashMap, project_root: AbsNormPathBuf, truncated: bool, strings: BTreeMap, @@ -126,8 +127,6 @@ struct BuildReportEntry { /// DO NOT UPDATE WITHOUT UPDATING `docs/users/build_observability/build_report.md`! #[derive(Debug, Clone, Serialize, PartialOrd, Ord, PartialEq, Eq)] struct BuildReportError { - // TODO(@wendyy) - remove `message` field - message: String, message_content: String, action_error: Option, /// An opaque index that can be use to de-duplicate errors. Two errors with the same @@ -146,7 +145,7 @@ enum EntryLabel { Target(TargetLabel), } -pub(crate) struct BuildReportCollector<'a> { +pub struct BuildReportCollector<'a> { artifact_fs: &'a ArtifactFs, overall_success: bool, include_unconfigured_section: bool, @@ -154,16 +153,20 @@ pub(crate) struct BuildReportCollector<'a> { error_cause_cache: HashMap, next_cause_index: usize, strings: BTreeMap, + failures: HashMap, + include_failures: bool, } impl<'a> BuildReportCollector<'a> { - pub(crate) fn convert( + pub fn convert( trace_id: &TraceId, artifact_fs: &'a ArtifactFs, project_root: &ProjectRoot, include_unconfigured_section: bool, include_other_outputs: bool, - build_result: &BuildTargetResult, + include_failures: bool, + configured: &BTreeMap>, + other_errors: &BTreeMap, Vec>, ) -> BuildReport { let mut this: BuildReportCollector<'_> = Self { artifact_fs, @@ -173,16 +176,12 @@ impl<'a> BuildReportCollector<'a> { error_cause_cache: HashMap::default(), next_cause_index: 0, strings: BTreeMap::default(), + failures: HashMap::default(), + include_failures, }; let mut entries = HashMap::new(); - if build_result - .other_errors - .values() - .flatten() - .next() - .is_some() - { + if other_errors.values().flatten().next().is_some() { // Do this check ahead of time. We don't check for errors that aren't associated // with a target below, so we'd miss this otherwise. this.overall_success = false; @@ -190,16 +189,14 @@ impl<'a> BuildReportCollector<'a> { // The `BuildTargetResult` doesn't group errors by their unconfigured target, so we need // to do a little iterator munging to achieve that ourselves - let results_by_unconfigured = &build_result - .configured + let results_by_unconfigured = configured .iter() .group_by(|x| x.0.target().unconfigured().dupe()); - let errors_by_unconfigured = build_result - .other_errors + let errors_by_unconfigured = other_errors .iter() .filter_map(|(l, e)| Some((l.as_ref()?.target().dupe(), e))); for i in Itertools::merge_join_by( - IntoIterator::into_iter(results_by_unconfigured), + IntoIterator::into_iter(&results_by_unconfigured), errors_by_unconfigured, |(l1, _), (l2, _)| Ord::cmp(l1, l2), ) { @@ -212,7 +209,7 @@ impl<'a> BuildReportCollector<'a> { (label, Either::Right(std::iter::empty()), &**errors) } }; - let entry = this.collect_results_for_unconfigured(results, errors); + let entry = this.collect_results_for_unconfigured(label.dupe(), results, errors); entries.insert(EntryLabel::Target(label), entry); } @@ -220,7 +217,7 @@ impl<'a> BuildReportCollector<'a> { trace_id: trace_id.dupe(), success: this.overall_success, results: entries, - failures: HashMap::new(), + failures: this.failures, project_root: project_root.root().to_owned(), // In buck1 we may truncate build report for a large number of targets. // Setting this to false since we don't currently truncate buck2's build report. @@ -240,6 +237,7 @@ impl<'a> BuildReportCollector<'a> { /// Always called for one unconfigured target at a time fn collect_results_for_unconfigured<'b>( &mut self, + target: TargetLabel, results: impl IntoIterator< Item = ( &'b ConfiguredProvidersLabel, @@ -261,7 +259,7 @@ impl<'a> BuildReportCollector<'a> { .filter_map(|(label, result)| Some((label, result.as_ref()?))) .group_by(|x| x.0.target().dupe()) { - let configured_report = self.collect_results_for_configured(results); + let configured_report = self.collect_results_for_configured(target.dupe(), results); if let Some(report) = unconfigured_report.as_mut() { if !configured_report.errors.is_empty() { report.success = BuildOutcome::FAIL; @@ -291,7 +289,7 @@ impl<'a> BuildReportCollector<'a> { configured_reports.insert(label.cfg().dupe(), configured_report); } - let errors = self.convert_error_list(errors); + let errors = self.convert_error_list(errors, target); if !errors.is_empty() { if let Some(report) = unconfigured_report.as_mut() { report.success = BuildOutcome::FAIL; @@ -307,6 +305,7 @@ impl<'a> BuildReportCollector<'a> { fn collect_results_for_configured<'b>( &mut self, + target: TargetLabel, results: impl IntoIterator< Item = ( &'b ConfiguredProvidersLabel, @@ -375,7 +374,7 @@ impl<'a> BuildReportCollector<'a> { configured_report.inner.configured_graph_size = Some(configured_graph_size); } } - configured_report.errors = self.convert_error_list(&errors); + configured_report.errors = self.convert_error_list(&errors, target); if !configured_report.errors.is_empty() { configured_report.inner.success = BuildOutcome::FAIL; } @@ -385,7 +384,11 @@ impl<'a> BuildReportCollector<'a> { /// Note: In order for production of the build report to be deterministic, the order in /// which this function is called, and which errors it is called with, must be /// deterministic. The particular order of the errors need not be. - fn convert_error_list(&mut self, errors: &[buck2_error::Error]) -> Vec { + fn convert_error_list( + &mut self, + errors: &[buck2_error::Error], + target: TargetLabel, + ) -> Vec { if errors.is_empty() { return Vec::new(); } @@ -460,13 +463,27 @@ impl<'a> BuildReportCollector<'a> { let message_content = self.update_string_cache(info.message.clone()); out.push(BuildReportError { - message: info.message, message_content, action_error: info.action_error, cause_index, }); } + if self.include_failures { + // Order is deterministic now, so picking the last one is fine. Also, we checked that + // there was at least one error above. + // + // This both omits errors and overwrites previous ones. That's the price you pay for + // using buck1 + self.failures.insert( + EntryLabel::Target(target), + self.strings + .get(&out.last().unwrap().message_content) + .unwrap() + .to_string(), + ); + } + out } } diff --git a/app/buck2_build_api/src/build/mod.rs b/app/buck2_build_api/src/build/mod.rs index fe86b0c48c3bd..105dae2a70d73 100644 --- a/app/buck2_build_api/src/build/mod.rs +++ b/app/buck2_build_api/src/build/mod.rs @@ -30,8 +30,10 @@ use dashmap::DashMap; use dice::DiceComputations; use dice::UserComputationData; use dupe::Dupe; +use dupe::OptionDupedExt; use futures::future; use futures::stream::BoxStream; +use futures::stream::FuturesOrdered; use futures::stream::FuturesUnordered; use futures::stream::Stream; use futures::stream::StreamExt; @@ -41,19 +43,25 @@ use tokio::sync::Mutex; use crate::actions::artifact::get_artifact_fs::GetArtifactFs; use crate::actions::artifact::materializer::ArtifactMaterializer; +use crate::actions::calculation::BuildKey; use crate::analysis::calculation::RuleAnalysisCalculation; use crate::artifact_groups::calculation::ArtifactGroupCalculation; +use crate::artifact_groups::calculation::EnsureTransitiveSetProjectionKey; use crate::artifact_groups::ArtifactGroup; use crate::artifact_groups::ArtifactGroupValues; +use crate::artifact_groups::ResolvedArtifactGroup; +use crate::artifact_groups::ResolvedArtifactGroupBuildSignalsKey; use crate::build_signals::HasBuildSignals; use crate::interpreter::rule_defs::cmd_args::AbsCommandLineContext; use crate::interpreter::rule_defs::cmd_args::CommandLineArgLike; use crate::interpreter::rule_defs::cmd_args::SimpleCommandLineArtifactVisitor; use crate::interpreter::rule_defs::provider::builtin::run_info::FrozenRunInfo; use crate::interpreter::rule_defs::provider::test_provider::TestProvider; +use crate::keep_going; +mod action_error; +pub mod build_report; mod graph_size; - /// The types of provider to build on the configured providers label #[derive(Debug, Clone, Dupe, Allocative)] pub enum BuildProviderType { @@ -377,13 +385,31 @@ async fn build_configured_label_inner<'a>( }; if let Some(signals) = ctx.per_transaction_data().get_build_signals() { - signals.top_level_target( - providers_label.target().dupe(), - outputs - .iter() - .map(|(output, _type)| output.dupe()) - .collect(), - ); + let resolved_artifact_futs: FuturesOrdered<_> = outputs + .iter() + .map(|(output, _type)| async move { output.resolved_artifact(ctx).await }) + .collect(); + + let resolved_artifacts: Vec<_> = + tokio::task::unconstrained(keep_going::try_join_all(ctx, resolved_artifact_futs)) + .await?; + let node_keys = resolved_artifacts + .iter() + .filter_map(|resolved| match resolved.dupe() { + ResolvedArtifactGroup::Artifact(artifact) => artifact + .action_key() + .duped() + .map(BuildKey) + .map(ResolvedArtifactGroupBuildSignalsKey::BuildKey), + ResolvedArtifactGroup::TransitiveSetProjection(key) => Some( + ResolvedArtifactGroupBuildSignalsKey::EnsureTransitiveSetProjectionKey( + EnsureTransitiveSetProjectionKey(key.dupe().dupe()), + ), + ), + }) + .collect(); + + signals.top_level_target(providers_label.target().dupe(), node_keys); } if !opts.skippable && outputs.is_empty() { diff --git a/app/buck2_build_api/src/build_signals.rs b/app/buck2_build_api/src/build_signals.rs index a1bfead86da1c..40e031fb11477 100644 --- a/app/buck2_build_api/src/build_signals.rs +++ b/app/buck2_build_api/src/build_signals.rs @@ -19,7 +19,7 @@ use dice::ActivationTracker; use dice::UserComputationData; use dupe::Dupe; -use crate::artifact_groups::ArtifactGroup; +use crate::artifact_groups::ResolvedArtifactGroupBuildSignalsKey; pub static CREATE_BUILD_SIGNALS: LateBinding< fn() -> (BuildSignalsInstaller, Box), @@ -38,7 +38,11 @@ pub struct BuildSignalsInstaller { /// Send notifications to the build signals backend. pub trait BuildSignals: Send + Sync + 'static { - fn top_level_target(&self, label: ConfiguredTargetLabel, artifacts: Vec); + fn top_level_target( + &self, + label: ConfiguredTargetLabel, + artifacts: Vec, + ); fn final_materialization( &self, diff --git a/app/buck2_build_api/src/bxl/build_result.rs b/app/buck2_build_api/src/bxl/build_result.rs index 54f452904256b..348e93ef74436 100644 --- a/app/buck2_build_api/src/bxl/build_result.rs +++ b/app/buck2_build_api/src/bxl/build_result.rs @@ -7,23 +7,44 @@ * of this source tree. */ +use std::fmt::Display; + use allocative::Allocative; +use buck2_core::provider::label::ConfiguredProvidersLabel; use gazebo::variants::UnpackVariants; use crate::build::ConfiguredBuildTargetResult; - -#[derive(Clone, Debug, derive_more::Display, UnpackVariants, Allocative)] +#[derive(Clone, Debug, UnpackVariants, Allocative)] pub enum BxlBuildResult { None, - #[display(fmt = "build result")] - Built(ConfiguredBuildTargetResult), + Built { + label: ConfiguredProvidersLabel, + result: ConfiguredBuildTargetResult, + }, } impl BxlBuildResult { - pub fn new(result: Option) -> Self { + pub fn new( + label: ConfiguredProvidersLabel, + result: Option, + ) -> Self { match result { - Some(result) => Self::Built(result), + Some(result) => Self::Built { label, result }, None => Self::None, } } } + +impl Display for BxlBuildResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BxlBuildResult::None => write!(f, "BxlBuildResult::None"), + BxlBuildResult::Built { result, .. } => write!( + f, + "BxlBuildResult::Built({} outputs, {} errors)", + result.outputs.len(), + result.errors.len() + ), + } + } +} diff --git a/app/buck2_build_api/src/bxl/mod.rs b/app/buck2_build_api/src/bxl/mod.rs index c00a4a907e6fe..b5a560d85034e 100644 --- a/app/buck2_build_api/src/bxl/mod.rs +++ b/app/buck2_build_api/src/bxl/mod.rs @@ -10,7 +10,6 @@ //! //! bxl is the Buck Extension Language, allowing any integrator to write Starlark code that //! introspects buck2 internal graphs in a safe, incremental way to perform more complex operations -//! pub mod build_result; diff --git a/app/buck2_build_api/src/bxl/result.rs b/app/buck2_build_api/src/bxl/result.rs index fb28daffce2cd..1684cd17ad72d 100644 --- a/app/buck2_build_api/src/bxl/result.rs +++ b/app/buck2_build_api/src/bxl/result.rs @@ -79,4 +79,18 @@ impl BxlResult { BxlResult::BuildsArtifacts { error_loc, .. } => error_loc, } } + + pub fn get_artifacts_opt(&self) -> Option<&Vec> { + match self { + BxlResult::None { .. } => None, + BxlResult::BuildsArtifacts { artifacts, .. } => Some(artifacts), + } + } + + pub fn get_build_result_opt(&self) -> Option<&Vec> { + match self { + BxlResult::None { .. } => None, + BxlResult::BuildsArtifacts { built, .. } => Some(built), + } + } } diff --git a/app/buck2_build_api/src/configure_targets.rs b/app/buck2_build_api/src/configure_targets.rs index 6d63d9dc7690e..932ba2bfc2c05 100644 --- a/app/buck2_build_api/src/configure_targets.rs +++ b/app/buck2_build_api/src/configure_targets.rs @@ -7,13 +7,13 @@ * of this source tree. */ +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::configuration::compatibility::IncompatiblePlatformReason; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::package::PackageLabel; use buck2_core::pattern::pattern_type::TargetPatternExtra; use buck2_core::pattern::ParsedPattern; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_core::target::label::TargetLabel; use buck2_events::dispatch::console_message; use buck2_node::load_patterns::load_patterns; use buck2_node::load_patterns::MissingTargetBehavior; @@ -56,7 +56,7 @@ fn split_compatible_incompatible( pub async fn get_maybe_compatible_targets<'a>( ctx: &'a DiceComputations, loaded_targets: impl IntoIterator>)>, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, keep_going: bool, ) -> anyhow::Result>>> { let mut by_package_fns: Vec<_> = Vec::new(); @@ -71,7 +71,7 @@ pub async fn get_maybe_compatible_targets<'a>( for<'x> |ctx: &'x mut DiceComputationsParallel<'a>| -> BoxFuture<'x, anyhow::Result>> { async move { let target = ctx - .get_configured_target(target.label(), global_target_platform) + .get_configured_target(target.label(), global_cfg_options) .await?; anyhow::Ok(ctx.get_configured_target_node(&target).await?) }.boxed() @@ -99,11 +99,10 @@ pub async fn get_maybe_compatible_targets<'a>( pub async fn get_compatible_targets( ctx: &DiceComputations, loaded_targets: impl IntoIterator>)>, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result> { let maybe_compatible_targets = - get_maybe_compatible_targets(ctx, loaded_targets, global_target_platform.as_ref(), false) - .await?; + get_maybe_compatible_targets(ctx, loaded_targets, global_cfg_options, false).await?; let (compatible_targets, incompatible_targets) = split_compatible_incompatible(maybe_compatible_targets)?; @@ -120,14 +119,14 @@ pub async fn get_compatible_targets( pub async fn load_compatible_patterns( ctx: &DiceComputations, parsed_patterns: Vec>, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, skip_missing_targets: MissingTargetBehavior, ) -> anyhow::Result> { let loaded_patterns = load_patterns(ctx, parsed_patterns, skip_missing_targets).await?; get_compatible_targets( ctx, loaded_patterns.iter_loaded_targets_by_package(), - global_target_platform, + global_cfg_options, ) .await } diff --git a/app/buck2_build_api/src/deferred/calculation.rs b/app/buck2_build_api/src/deferred/calculation.rs index edb7f44b2b61c..105b1e3c2d15c 100644 --- a/app/buck2_build_api/src/deferred/calculation.rs +++ b/app/buck2_build_api/src/deferred/calculation.rs @@ -49,6 +49,7 @@ use crate::actions::artifact::get_artifact_fs::GetArtifactFs; use crate::analysis::calculation::RuleAnalysisCalculation; use crate::analysis::AnalysisResult; use crate::artifact_groups::calculation::ArtifactGroupCalculation; +use crate::artifact_groups::promise::PromiseArtifact; use crate::artifact_groups::ArtifactGroup; use crate::bxl::calculation::BXL_CALCULATION_IMPL; use crate::bxl::result::BxlResult; @@ -107,6 +108,13 @@ pub static EVAL_ANON_TARGET: LateBinding< ) -> Pin> + Send + 'c>>, > = LateBinding::new("EVAL_ANON_TARGET"); +pub static GET_PROMISED_ARTIFACT: LateBinding< + for<'c> fn( + &'c PromiseArtifact, + &'c DiceComputations, + ) -> Pin> + Send + 'c>>, +> = LateBinding::new("GET_PROMISED_ARTIFACT"); + async fn lookup_deferred_inner( key: &BaseDeferredKey, dice: &DiceComputations, diff --git a/app/buck2_build_api/src/interpreter/more.rs b/app/buck2_build_api/src/interpreter/more.rs index b7ff0a7bfaa3a..8e71475f193de 100644 --- a/app/buck2_build_api/src/interpreter/more.rs +++ b/app/buck2_build_api/src/interpreter/more.rs @@ -10,6 +10,7 @@ use buck2_interpreter::functions::more::REGISTER_BUCK2_BUILD_API_GLOBALS; use starlark::environment::GlobalsBuilder; +use crate::actions::error_handler::register_action_error_handler_for_testing; use crate::actions::error_handler::register_action_error_types; use crate::interpreter::rule_defs::artifact::artifact_type::register_artifact; use crate::interpreter::rule_defs::artifact::starlark_artifact_value::register_artifact_value; @@ -44,6 +45,7 @@ fn register_build_api_globals(globals: &mut GlobalsBuilder) { register_artifact_value(globals); register_output_artifact(globals); register_action_error_types(globals); + register_action_error_handler_for_testing(globals); } pub(crate) fn init_register_build_api_globals() { diff --git a/app/buck2_build_api/src/interpreter/rule_defs/artifact/starlark_promise_artifact.rs b/app/buck2_build_api/src/interpreter/rule_defs/artifact/starlark_promise_artifact.rs index de8a7a82baa86..10e9620bbd056 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/artifact/starlark_promise_artifact.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/artifact/starlark_promise_artifact.rs @@ -91,7 +91,7 @@ enum PromiseArtifactError { pub struct StarlarkPromiseArtifact { pub declaration_location: Option, pub artifact: PromiseArtifact, - short_path: Option, + pub short_path: Option, } starlark_simple_value!(StarlarkPromiseArtifact); diff --git a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/mod.rs b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/mod.rs index 948d132a3c08a..f5dfce6f73504 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/mod.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/mod.rs @@ -11,6 +11,7 @@ pub mod arg_builder; mod builder; mod options; pub(crate) mod regex; +pub(crate) mod shlex_quote; pub mod space_separated; mod traits; mod typ; diff --git a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/options.rs b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/options.rs index 1f81b194ef9ad..6faf82d9d6440 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/options.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/options.rs @@ -50,6 +50,7 @@ use crate::interpreter::rule_defs::artifact::StarlarkArtifactLike; use crate::interpreter::rule_defs::artifact::ValueAsArtifactLike; use crate::interpreter::rule_defs::cmd_args::regex::CmdArgsRegex; use crate::interpreter::rule_defs::cmd_args::regex::FrozenCmdArgsRegex; +use crate::interpreter::rule_defs::cmd_args::shlex_quote::shlex_quote; use crate::interpreter::rule_defs::cmd_args::traits::CommandLineContext; use crate::interpreter::rule_defs::cmd_args::CommandLineBuilder; use crate::interpreter::rule_defs::cmd_args::CommandLineLocation; @@ -645,7 +646,8 @@ impl<'v, 'x> CommandLineOptionsRef<'v, 'x> { } match &self.opts.quote { Some(QuoteStyle::Shell) => { - arg = shlex::quote(&arg).into_owned(); + let quoted = shlex_quote(&arg); + arg = quoted.into_owned(); } _ => {} } diff --git a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/shlex_quote.rs b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/shlex_quote.rs new file mode 100644 index 0000000000000..4076385058d13 --- /dev/null +++ b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/shlex_quote.rs @@ -0,0 +1,62 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::borrow::Cow; + +/// Quote string for shell. +/// +/// This is copy-paste from [`shlex`](https://github.com/comex/rust-shlex/) 1.0. +/// +/// Generally shell quoting is unspecified, for example `a` can be quoted as: +/// - `a` +/// - `"a"` +/// - `'a'` +/// - and many more +/// +/// to be used in shell. But we use shell quoting to generate arguments for argfiles. +/// And certain programs expect certain quoting style (for example, `cl.exe` expect double-quoted). +/// Additionally, we probably also incorrectly use shell quoting for `cmd.exe`. +/// +/// Long story short, we should not depend on possible correct behavior change in `shlex` crate. +pub(crate) fn shlex_quote(in_str: &str) -> Cow { + if in_str.is_empty() { + "\"\"".into() + } else if in_str.bytes().any(|c| match c as char { + '|' | '&' | ';' | '<' | '>' | '(' | ')' | '$' | '`' | '\\' | '"' | '\'' | ' ' | '\t' + | '\r' | '\n' | '*' | '?' | '[' | '#' | '~' | '=' | '%' => true, + _ => false, + }) { + let mut out: Vec = Vec::new(); + out.push(b'"'); + for c in in_str.bytes() { + match c as char { + '$' | '`' | '"' | '\\' => out.push(b'\\'), + _ => (), + } + out.push(c); + } + out.push(b'"'); + unsafe { String::from_utf8_unchecked(out) }.into() + } else { + in_str.into() + } +} + +#[cfg(test)] +mod tests { + use crate::interpreter::rule_defs::cmd_args::shlex_quote::shlex_quote; + + #[test] + fn test_quote() { + assert_eq!(shlex_quote("foobar"), "foobar"); + assert_eq!(shlex_quote("foo bar"), "\"foo bar\""); + assert_eq!(shlex_quote("\""), "\"\\\"\""); + assert_eq!(shlex_quote(""), "\"\""); + } +} diff --git a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/typ.rs b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/typ.rs index f298c38b5fa4b..782385d97c345 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/typ.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/cmd_args/typ.rs @@ -694,6 +694,11 @@ fn cmd_args_methods(builder: &mut MethodsBuilder) { /// Make all artifact paths relative to a given location. Typically used when the command /// you are running changes directory. /// + /// By default, the paths are relative to the artifacts themselves (equivalent to + /// `parent = 0`). Use `parent` to make the paths relative to an ancestor directory. + /// For example `parent = 1` would make all paths relative to the containing dirs + /// of any artifacts in the `cmd_args`. + /// /// ```python /// dir = symlinked_dir(...) /// script = [ diff --git a/app/buck2_build_api/src/interpreter/rule_defs/context.rs b/app/buck2_build_api/src/interpreter/rule_defs/context.rs index ef2187a3d14f7..e31a0c1ca35b8 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/context.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/context.rs @@ -14,15 +14,11 @@ use std::fmt::Display; use std::fmt::Formatter; use allocative::Allocative; -use buck2_artifact::artifact::artifact_type::Artifact; -use buck2_core::base_deferred_key::BaseDeferredKey; -use buck2_error::Context; use buck2_execute::digest_config::DigestConfig; use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use buck2_util::late_binding::LateBinding; use derive_more::Display; use dice::DiceComputations; -use dupe::Dupe; use starlark::any::ProvidesStaticType; use starlark::environment::GlobalsBuilder; use starlark::environment::Methods; @@ -47,9 +43,7 @@ use starlark::values::ValueTyped; use starlark::values::ValueTypedComplex; use crate::analysis::registry::AnalysisRegistry; -use crate::artifact_groups::promise::PromiseArtifact; -use crate::artifact_groups::promise::PromiseArtifactResolveError; -use crate::deferred::calculation::EVAL_ANON_TARGET; +use crate::deferred::calculation::GET_PROMISED_ARTIFACT; use crate::interpreter::rule_defs::plugins::AnalysisPlugins; /// Functions to allow users to interact with the Actions registry. @@ -121,7 +115,7 @@ impl<'v> AnalysisActions<'v> { }; for consumer_artifact in consumer_analysis_artifacts { - let artifact = get_artifact_from_anon_target_analysis(&consumer_artifact, dice).await?; + let artifact = (GET_PROMISED_ARTIFACT.get()?)(&consumer_artifact, dice).await?; let short_path = short_path_assertions.get(consumer_artifact.id()).cloned(); consumer_artifact.resolve(artifact.clone(), &short_path)?; } @@ -129,34 +123,6 @@ impl<'v> AnalysisActions<'v> { } } -pub async fn get_artifact_from_anon_target_analysis( - promise_artifact: &PromiseArtifact, - dice: &DiceComputations, -) -> anyhow::Result { - let promise_id = promise_artifact.id(); - let owner = promise_artifact.owner(); - let analysis_result = match owner { - BaseDeferredKey::AnonTarget(anon_target) => { - (EVAL_ANON_TARGET.get()?)(dice, anon_target.dupe()).await? - } - _ => { - return Err(PromiseArtifactResolveError::OwnerIsNotAnonTarget( - promise_artifact.clone(), - owner.clone(), - ) - .into()); - } - }; - - analysis_result - .promise_artifact_map() - .get(promise_id) - .context(PromiseArtifactResolveError::NotFoundInAnalysis( - promise_artifact.clone(), - )) - .cloned() -} - #[starlark_value(type = "actions", StarlarkTypeRepr, UnpackValue)] impl<'v> StarlarkValue<'v> for AnalysisActions<'v> { fn get_methods() -> Option<&'static Methods> { diff --git a/app/buck2_build_api/src/query/bxl.rs b/app/buck2_build_api/src/query/bxl.rs index 79db98dfb23f4..2bbbe7f8640b0 100644 --- a/app/buck2_build_api/src/query/bxl.rs +++ b/app/buck2_build_api/src/query/bxl.rs @@ -11,13 +11,13 @@ use std::future::Future; use std::pin::Pin; use async_trait::async_trait; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::name::CellName; use buck2_core::cells::CellResolver; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::fs::project::ProjectRoot; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_core::target::label::TargetLabel; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_node::nodes::unconfigured::TargetNode; use buck2_query::query::syntax::simple::eval::file_set::FileSet; @@ -171,8 +171,8 @@ pub trait BxlAqueryFunctions: Send { pub static NEW_BXL_CQUERY_FUNCTIONS: LateBinding< fn( - // Target platform - Option, + // Target configuration info (target platform + cli modifiers) + GlobalCfgOptions, ProjectRoot, CellName, CellResolver, @@ -189,8 +189,8 @@ pub static NEW_BXL_UQUERY_FUNCTIONS: LateBinding< pub static NEW_BXL_AQUERY_FUNCTIONS: LateBinding< fn( - // Target platform - Option, + // Target configuration info (target platform + cli modifiers) + GlobalCfgOptions, ProjectRoot, CellName, CellResolver, diff --git a/app/buck2_build_api/src/query/oneshot.rs b/app/buck2_build_api/src/query/oneshot.rs index 35f4e402a3720..2ab0389800e3e 100644 --- a/app/buck2_build_api/src/query/oneshot.rs +++ b/app/buck2_build_api/src/query/oneshot.rs @@ -8,8 +8,8 @@ */ use async_trait::async_trait; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_node::configured_universe::CqueryUniverse; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_node::nodes::unconfigured::TargetNode; @@ -35,7 +35,7 @@ pub trait QueryFrontend: Send + Sync + 'static { working_dir: &ProjectRelativePath, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result>; async fn eval_cquery( @@ -45,7 +45,7 @@ pub trait QueryFrontend: Send + Sync + 'static { owner_behavior: CqueryOwnerBehavior, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, target_universe: Option<&[String]>, ) -> anyhow::Result>; @@ -55,7 +55,7 @@ pub trait QueryFrontend: Send + Sync + 'static { working_dir: &ProjectRelativePath, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result>; async fn universe_from_literals( @@ -63,7 +63,7 @@ pub trait QueryFrontend: Send + Sync + 'static { ctx: &DiceComputations, cwd: &ProjectRelativePath, literals: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result; } diff --git a/app/buck2_build_api/src/transition.rs b/app/buck2_build_api/src/transition.rs index c63d4ddc6f032..d21ccb6ecff33 100644 --- a/app/buck2_build_api/src/transition.rs +++ b/app/buck2_build_api/src/transition.rs @@ -13,7 +13,7 @@ use async_trait::async_trait; use buck2_core::configuration::data::ConfigurationData; use buck2_core::configuration::transition::applied::TransitionApplied; use buck2_core::configuration::transition::id::TransitionId; -use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeRef; use buck2_util::late_binding::LateBinding; use dice::DiceComputations; @@ -23,7 +23,7 @@ pub trait TransitionCalculation: Send + Sync + 'static { async fn apply_transition( &self, ctx: &DiceComputations, - target_node: &TargetNode, + target_node: TargetNodeRef<'_>, conf: &ConfigurationData, transition_id: &TransitionId, ) -> anyhow::Result>; diff --git a/app/buck2_build_api_tests/src/actions/calculation.rs b/app/buck2_build_api_tests/src/actions/calculation.rs index cbbde0d5608ef..c1926c7eaae71 100644 --- a/app/buck2_build_api_tests/src/actions/calculation.rs +++ b/app/buck2_build_api_tests/src/actions/calculation.rs @@ -145,10 +145,7 @@ fn mock_deferred_resolution_calculation( ) -> DiceBuilder { let arc_any: Arc = Arc::new(registered_action_arc); let an_any = DeferredValueAnyReady::AnyValue(arc_any); - dice_builder.mock_and_return( - deferred_resolve, - anyhow::Ok(an_any).map_err(buck2_error::Error::from), - ) + dice_builder.mock_and_return(deferred_resolve, buck2_error::Ok(an_any)) } async fn make_default_dice_state( diff --git a/app/buck2_build_api_tests/src/deferred/calculation.rs b/app/buck2_build_api_tests/src/deferred/calculation.rs index 6333e8c56da5a..0157640497db8 100644 --- a/app/buck2_build_api_tests/src/deferred/calculation.rs +++ b/app/buck2_build_api_tests/src/deferred/calculation.rs @@ -101,13 +101,12 @@ async fn lookup_deferred_from_analysis() -> anyhow::Result<()> { }) .mock_and_return( analysis_key, - anyhow::Ok(MaybeCompatible::Compatible(AnalysisResult::new( + buck2_error::Ok(MaybeCompatible::Compatible(AnalysisResult::new( provider_collection, deferred_result, None, HashMap::new(), - ))) - .map_err(buck2_error::Error::from), + ))), ) .mock_and_return( configured_node_key, @@ -196,13 +195,12 @@ async fn lookup_deferred_that_has_deferreds() -> anyhow::Result<()> { }) .mock_and_return( analysis_key, - anyhow::Ok(MaybeCompatible::Compatible(AnalysisResult::new( + buck2_error::Ok(MaybeCompatible::Compatible(AnalysisResult::new( provider_collection, deferred_result, None, HashMap::new(), - ))) - .map_err(buck2_error::Error::from), + ))), ) .mock_and_return( configured_node_key, diff --git a/app/buck2_build_api_tests/src/interpreter/rule_defs/artifact/testing.rs b/app/buck2_build_api_tests/src/interpreter/rule_defs/artifact/testing.rs index 108518ae0f17b..53c90cd521a3d 100644 --- a/app/buck2_build_api_tests/src/interpreter/rule_defs/artifact/testing.rs +++ b/app/buck2_build_api_tests/src/interpreter/rule_defs/artifact/testing.rs @@ -182,7 +182,7 @@ pub(crate) fn artifactory(builder: &mut GlobalsBuilder) { .add_to_command_line(&mut cli, &mut ctx) .unwrap(); assert_eq!(1, cli.len()); - Ok(cli.get(0).unwrap().to_owned()) + Ok(cli.first().unwrap().to_owned()) } // Mainly tests get_or_declare_output function that can transfer associated artifacts diff --git a/app/buck2_build_api_tests/src/interpreter/rule_defs/cmd_args/testing.rs b/app/buck2_build_api_tests/src/interpreter/rule_defs/cmd_args/testing.rs index e64f73972e8f2..20d12267452c8 100644 --- a/app/buck2_build_api_tests/src/interpreter/rule_defs/cmd_args/testing.rs +++ b/app/buck2_build_api_tests/src/interpreter/rule_defs/cmd_args/testing.rs @@ -64,6 +64,6 @@ pub(crate) fn command_line_stringifier(builder: &mut GlobalsBuilder) { .0 .add_to_command_line(&mut cli, &mut ctx)?; assert_eq!(1, cli.len()); - Ok(cli.get(0).unwrap().clone()) + Ok(cli.first().unwrap().clone()) } } diff --git a/app/buck2_build_api_tests/src/nodes/calculation.rs b/app/buck2_build_api_tests/src/nodes/calculation.rs index e677d2ef920cf..acc10c9f65c4b 100644 --- a/app/buck2_build_api_tests/src/nodes/calculation.rs +++ b/app/buck2_build_api_tests/src/nodes/calculation.rs @@ -138,10 +138,10 @@ async fn test_get_node() -> anyhow::Result<()> { let computations = computations.commit().await; let node = computations.get_target_node(&label1).await?; - assert_eq!(node.0, node1.0); + assert_eq!(node, node1); let node = computations.get_target_node(&label2).await?; - assert_eq!(node.0, node2.0); + assert_eq!(node, node2); let conf_attrs1 = smallmap![ "bool_field" => ConfiguredAttr::Bool(BoolLiteral(false)), @@ -167,10 +167,10 @@ async fn test_get_node() -> anyhow::Result<()> { ]; let node = computations.get_target_node(&label1).await?; - assert_eq!(node.0, node1.0); + assert_eq!(node, node1); let node = computations.get_target_node(&label2).await?; - assert_eq!(node.0, node2.0); + assert_eq!(node, node2); let node = computations .get_configured_target_node(&label1.configure(cfg.dupe())) diff --git a/app/buck2_build_signals/src/lib.rs b/app/buck2_build_signals/src/lib.rs index 9035d2419bf96..c2de5b3152f47 100644 --- a/app/buck2_build_signals/src/lib.rs +++ b/app/buck2_build_signals/src/lib.rs @@ -30,6 +30,8 @@ pub struct NodeDuration { pub user: Duration, /// The total duration for this node. pub total: Duration, + /// The waiting duration for this node. + pub queue: Option, } impl NodeDuration { @@ -44,6 +46,7 @@ impl NodeDuration { Self { user: Duration::from_secs(0), total: Duration::from_secs(0), + queue: None, } } } diff --git a/app/buck2_build_signals_impl/src/lib.rs b/app/buck2_build_signals_impl/src/lib.rs index e5add738258be..f2fcb7d328065 100644 --- a/app/buck2_build_signals_impl/src/lib.rs +++ b/app/buck2_build_signals_impl/src/lib.rs @@ -27,8 +27,7 @@ use buck2_build_api::actions::calculation::BuildKeyActivationData; use buck2_build_api::actions::RegisteredAction; use buck2_build_api::artifact_groups::calculation::EnsureProjectedArtifactKey; use buck2_build_api::artifact_groups::calculation::EnsureTransitiveSetProjectionKey; -use buck2_build_api::artifact_groups::ArtifactGroup; -use buck2_build_api::artifact_groups::ResolvedArtifactGroup; +use buck2_build_api::artifact_groups::ResolvedArtifactGroupBuildSignalsKey; use buck2_build_api::build_signals::BuildSignals; use buck2_build_api::build_signals::BuildSignalsInstaller; use buck2_build_api::build_signals::CREATE_BUILD_SIGNALS; @@ -56,7 +55,7 @@ use derive_more::From; use dice::ActivationData; use dice::ActivationTracker; use dupe::Dupe; -use dupe::OptionDupedExt; +use gazebo::prelude::SliceExt; use itertools::Itertools; use smallvec::SmallVec; use static_assertions::assert_eq_size; @@ -154,7 +153,7 @@ impl fmt::Display for NodeKey { struct TopLevelTargetSignal { pub label: ConfiguredTargetLabel, - pub artifacts: Vec, + pub artifacts: Vec, } struct FinalMaterializationSignal { @@ -203,7 +202,11 @@ pub struct BuildSignalSender { } impl BuildSignals for BuildSignalSender { - fn top_level_target(&self, label: ConfiguredTargetLabel, artifacts: Vec) { + fn top_level_target( + &self, + label: ConfiguredTargetLabel, + artifacts: Vec, + ) { let _ignored = self .sender .send(TopLevelTargetSignal { label, artifacts }.into()); @@ -278,6 +281,7 @@ impl ActivationTracker for BuildSignalSender { signal.duration = NodeDuration { user: duration, total: duration, + queue: None, }; signal.spans = spans; } else if let Some(IntepreterResultsKeyActivationData { @@ -289,6 +293,7 @@ impl ActivationTracker for BuildSignalSender { signal.duration = NodeDuration { user: duration, total: duration, + queue: None, }; signal.load_result = result.ok(); @@ -299,6 +304,7 @@ impl ActivationTracker for BuildSignalSender { signal.duration = NodeDuration { user: duration, total: duration, + queue: None, }; signal.spans = spans; } @@ -414,6 +420,7 @@ where duration: NodeDuration { user: Duration::ZERO, total: compute_elapsed, + queue: None, }, span_ids: Default::default(), }; @@ -485,6 +492,7 @@ where .collect(), duration: Some(data.duration.critical_path_duration().try_into()?), user_duration: Some(data.duration.user.try_into()?), + queue_duration: data.duration.queue.map(|d| d.try_into()).transpose()?, total_duration: Some(data.duration.total.try_into()?), potential_improvement_duration: potential_improvement .map(|p| p.try_into()) @@ -566,26 +574,14 @@ where &mut self, top_level: TopLevelTargetSignal, ) -> Result<(), anyhow::Error> { - let artifact_keys = - top_level - .artifacts - .into_iter() - .filter_map(|dep| match dep.assert_resolved() { - ResolvedArtifactGroup::Artifact(artifact) => artifact - .action_key() - .duped() - .map(BuildKey) - .map(NodeKey::BuildKey), - ResolvedArtifactGroup::TransitiveSetProjection(key) => { - Some(NodeKey::EnsureTransitiveSetProjectionKey( - EnsureTransitiveSetProjectionKey(key.dupe()), - )) - } - }); - self.backend.process_top_level_target( NodeKey::AnalysisKey(AnalysisKey(top_level.label)), - artifact_keys, + top_level.artifacts.map(|k| match k { + ResolvedArtifactGroupBuildSignalsKey::BuildKey(b) => NodeKey::BuildKey(b.clone()), + ResolvedArtifactGroupBuildSignalsKey::EnsureTransitiveSetProjectionKey(e) => { + NodeKey::EnsureTransitiveSetProjectionKey(e.clone()) + } + }), ); Ok(()) @@ -623,7 +619,7 @@ struct NodeData { span_ids: SmallVec<[SpanId; 1]>, } -assert_eq_size!(NodeData, [usize; 8]); +assert_eq_size!(NodeData, [usize; 10]); fn create_build_signals() -> (BuildSignalsInstaller, Box) { let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); diff --git a/app/buck2_bxl/BUCK b/app/buck2_bxl/BUCK index 08a75491f8403..7030ce27ef0d6 100644 --- a/app/buck2_bxl/BUCK +++ b/app/buck2_bxl/BUCK @@ -16,7 +16,6 @@ rust_library( deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:async-recursion", - "fbsource//third-party/rust:async-scoped", "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:clap-3", "fbsource//third-party/rust:dashmap", diff --git a/app/buck2_bxl/Cargo.toml b/app/buck2_bxl/Cargo.toml index b7f7976ed9802..54dce123b847d 100644 --- a/app/buck2_bxl/Cargo.toml +++ b/app/buck2_bxl/Cargo.toml @@ -9,7 +9,6 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } async-recursion = { workspace = true } -async-scoped = { workspace = true } async-trait = { workspace = true } clap = { workspace = true } dashmap = { workspace = true } diff --git a/app/buck2_bxl/src/bxl/deferred.rs b/app/buck2_bxl/src/bxl/deferred.rs index c92c3031e5a9f..82b5cbcec3867 100644 --- a/app/buck2_bxl/src/bxl/deferred.rs +++ b/app/buck2_bxl/src/bxl/deferred.rs @@ -29,6 +29,7 @@ mod tests { use buck2_build_api::deferred::types::DeferredTable; use buck2_build_api::deferred::types::DeferredValue; use buck2_common::dice::data::testing::SetTestingIoProvider; + use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::base_deferred_key::BaseDeferredKey; use buck2_core::execution_types::execution::ExecutionPlatformResolution; use buck2_core::execution_types::executor_config::CommandExecutorConfig; @@ -80,8 +81,8 @@ mod tests { name: "foo".to_owned(), }, Arc::new(OrderedMap::new()), - None, false, + GlobalCfgOptions::default(), ); let mut deferred = DeferredRegistry::new(BaseKey::Base(BaseDeferredKey::BxlLabel( @@ -106,7 +107,7 @@ mod tests { }) .mock_and_return( BxlComputeKey(bxl.dupe()), - anyhow::Ok(BxlComputeResult { + buck2_error::Ok(BxlComputeResult { bxl_result: Arc::new(BxlResult::BuildsArtifacts { output_loc: mk_stream_cache("test", &bxl), error_loc: mk_stream_cache("errortest", &bxl), @@ -115,8 +116,7 @@ mod tests { deferred: deferred_result, }), materializations: Arc::new(Default::default()), - }) - .map_err(buck2_error::Error::from), + }), ); let mut dice_data = UserComputationData::new(); diff --git a/app/buck2_bxl/src/bxl/eval.rs b/app/buck2_bxl/src/bxl/eval.rs index a26cb5f345338..308313bee5cd6 100644 --- a/app/buck2_bxl/src/bxl/eval.rs +++ b/app/buck2_bxl/src/bxl/eval.rs @@ -19,6 +19,7 @@ use buck2_build_api::deferred::types::DeferredTable; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::data::HasIoProvider; use buck2_common::events::HasEvents; +use buck2_common::scope::scope_and_collect_with_dice; use buck2_common::target_aliases::BuckConfigTargetAliasResolver; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::base_deferred_key::BaseDeferredKey; @@ -33,7 +34,6 @@ use buck2_data::StarlarkFailNoStacktrace; use buck2_events::dispatch::console_message; use buck2_events::dispatch::get_dispatcher; use buck2_events::dispatch::with_dispatcher; -use buck2_events::dispatch::with_dispatcher_async; use buck2_events::dispatch::EventDispatcher; use buck2_execute::digest_config::HasDigestConfig; use buck2_futures::cancellable_future::CancellationObserver; @@ -98,17 +98,14 @@ pub(crate) async fn eval( // on the scope will be dropped at the earliest await point. If we are within the blocking // section of bxl, the cancellation observer will be notified and cause the blocking calls // to terminate. - async_scoped::TokioScope::scope_and_collect(|s| { + scope_and_collect_with_dice(ctx, |ctx, s| { s.spawn_cancellable( - with_dispatcher_async( - dispatcher.dupe(), - eval_bxl_inner( - ctx, - dispatcher, - key, - profile_mode_or_instrumentation, - liveness, - ), + eval_bxl_inner( + ctx, + dispatcher, + key, + profile_mode_or_instrumentation, + liveness, ), || Err(anyhow::anyhow!("cancelled")), ) @@ -193,8 +190,6 @@ async fn eval_bxl_inner( Some(profiler) => StarlarkProfilerOrInstrumentation::for_profiler(profiler), }; - let global_target_platform = key.global_target_platform().clone(); - let (bxl_result, materializations) = with_starlark_eval_provider( ctx, &mut profiler, @@ -231,7 +226,6 @@ async fn eval_bxl_inner( file, error_file, digest_config, - global_target_platform, )?; let bxl_ctx = ValueTyped::::new(env.heap().alloc(bxl_ctx)).unwrap(); diff --git a/app/buck2_bxl/src/bxl/key.rs b/app/buck2_bxl/src/bxl/key.rs index eccefd8ea3168..0a8a42ac097af 100644 --- a/app/buck2_bxl/src/bxl/key.rs +++ b/app/buck2_bxl/src/bxl/key.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use allocative::Allocative; use anyhow::Context; use buck2_build_api::bxl::types::BxlFunctionLabel; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::base_deferred_key::BaseDeferredKeyDyn; use buck2_core::execution_types::execution::ExecutionPlatformResolution; use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; @@ -23,7 +24,6 @@ use buck2_core::fs::project_rel_path::ProjectRelativePath; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_core::target::label::TargetLabel; use buck2_data::action_key_owner::BaseDeferredKeyProto; use buck2_data::ToProtoMessage; use cmp_any::PartialEqAny; @@ -50,14 +50,14 @@ impl BxlKey { pub(crate) fn new( spec: BxlFunctionLabel, bxl_args: Arc>, - global_target_platform: Option, force_print_stacktrace: bool, + global_cfg_options: GlobalCfgOptions, ) -> Self { Self(Arc::new(BxlKeyData { spec, bxl_args, - global_target_platform, force_print_stacktrace, + global_cfg_options, })) } @@ -91,8 +91,8 @@ impl BxlKey { .context("Not BxlKey (internal error)") } - pub(crate) fn global_target_platform(&self) -> &Option { - &self.0.global_target_platform + pub(crate) fn global_cfg_options(&self) -> &GlobalCfgOptions { + &self.0.global_cfg_options } pub(crate) fn force_print_stacktrace(&self) -> bool { @@ -115,11 +115,11 @@ impl BxlKey { struct BxlKeyData { spec: BxlFunctionLabel, bxl_args: Arc>, - global_target_platform: Option, /// Overrides `fail_no_stacktrace` to print a stacktrace anyway. FIXME(JakobDegen): Might be /// better to put this on the `UserComputationData` instead, to keep this from invalidating the /// dice node. A bit hard to wire up though, so just leave it here for now. force_print_stacktrace: bool, + global_cfg_options: GlobalCfgOptions, } impl BxlKeyData { @@ -186,7 +186,7 @@ impl BaseDeferredKeyDyn for BxlDynamicKeyData { let output_hash = { let mut hasher = DefaultHasher::new(); self.key.bxl_args.hash(&mut hasher); - self.key.global_target_platform.hash(&mut hasher); + self.key.global_cfg_options.hash(&mut hasher); let output_hash = hasher.finish(); format!("{:x}", output_hash) }; diff --git a/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs b/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs index d540395c1a244..8cd2a0d784d8a 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs @@ -88,7 +88,6 @@ fn starlark_analysis_result_methods(builder: &mut MethodsBuilder) { /// transitions. This means that you cannot create an exec dep or toolchain from an analysis result. /// We may support other dependency transition types in the future. - /// /// This is useful for passing in the results of `ctx.analysis()` into anon targets. /// diff --git a/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs b/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs index a0a301642cccf..831a7780b1440 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs @@ -12,12 +12,12 @@ use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::query::bxl::BxlAqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_AQUERY_FUNCTIONS; use buck2_build_api::query::oneshot::QUERY_FRONTEND; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::configuration::compatibility::IncompatiblePlatformReason; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::label::TargetLabel; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_query::query::syntax::simple::eval::set::TargetSet; -use buck2_query::query::syntax::simple::eval::set::TargetSetExt; use buck2_query::query::syntax::simple::functions::helpers::CapturedExpr; use derivative::Derivative; use derive_more::Display; @@ -72,7 +72,8 @@ pub(crate) struct StarlarkAQueryCtx<'v> { #[derivative(Debug = "ignore")] ctx: &'v BxlContext<'v>, #[derivative(Debug = "ignore")] - target_platform: Option, + // Overrides the GlobalCfgOptions in the BxlContext + global_cfg_options_override: GlobalCfgOptions, } #[starlark_value(type = "aqueryctx", StarlarkTypeRepr, UnpackValue)] @@ -96,28 +97,31 @@ impl<'v> StarlarkAQueryCtx<'v> { default_target_platform: &Option, ) -> anyhow::Result> { let target_platform = global_target_platform.parse_target_platforms( - &ctx.data.target_alias_resolver, - &ctx.data.cell_resolver, - ctx.data.cell_name, + ctx.target_alias_resolver(), + ctx.cell_resolver(), + ctx.cell_name(), default_target_platform, )?; Ok(Self { ctx, - target_platform, + global_cfg_options_override: GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, }) } } pub(crate) async fn get_aquery_env( ctx: &BxlContextNoDice<'_>, - target_platform: Option, + global_cfg_options_override: &GlobalCfgOptions, ) -> anyhow::Result> { (NEW_BXL_AQUERY_FUNCTIONS.get()?)( - target_platform, + global_cfg_options_override.clone(), ctx.project_root().dupe(), - ctx.cell_name, - ctx.cell_resolver.dupe(), + ctx.cell_name(), + ctx.cell_resolver().dupe(), ) .await } @@ -135,19 +139,18 @@ enum UnpackActionNodes<'v> { // and `ProvidersExpr`, we need to pass the aquery delegate a list of configured providers labels, and it will // run analysis on them to construct the `ActionQueryNode`s. async fn unpack_action_nodes<'v>( - expr: UnpackActionNodes<'v>, - target_platform: &Option, - ctx: &BxlContextNoDice<'v>, + this: &StarlarkAQueryCtx<'v>, dice: &mut DiceComputations, - aquery_env: &dyn BxlAqueryFunctions, + expr: UnpackActionNodes<'v>, ) -> anyhow::Result> { + let aquery_env = get_aquery_env(&this.ctx.data, &this.global_cfg_options_override).await?; let providers = match expr { UnpackActionNodes::ActionQueryNodes(action_nodes) => return Ok(action_nodes.0.clone()), UnpackActionNodes::ConfiguredProviders(arg) => { ProvidersExpr::::unpack( arg, - target_platform.clone(), - ctx, + &this.global_cfg_options_override, + &this.ctx.data, dice, ) .await? @@ -158,8 +161,8 @@ async fn unpack_action_nodes<'v>( UnpackActionNodes::ConfiguredTargets(arg) => { TargetListExpr::::unpack_opt( arg, - target_platform, - ctx, + &this.global_cfg_options_override, + &this.ctx.data, dice, true, ) @@ -171,14 +174,13 @@ async fn unpack_action_nodes<'v>( let (incompatible_targets, result) = aquery_env.get_target_set(dice, providers).await?; if !incompatible_targets.is_empty() { - ctx.print_to_error_stream(IncompatiblePlatformReason::skipping_message_for_multiple( - incompatible_targets.iter(), - ))?; + this.ctx.data.print_to_error_stream( + IncompatiblePlatformReason::skipping_message_for_multiple(incompatible_targets.iter()), + )?; } Ok(result) } - /// The context for performing `aquery` operations in bxl. The functions offered on this ctx are /// the same behaviour as the query functions available within aquery command. /// @@ -198,21 +200,14 @@ fn aquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let aquery_env = get_aquery_env(ctx, this.target_platform.dupe()).await?; - let filter = filter .into_option() .try_map(buck2_query_parser::parse_expr)?; - let universe = unpack_action_nodes( - universe, - &this.target_platform, - ctx, - dice, - aquery_env.as_ref(), - ) - .await?; + let universe = unpack_action_nodes(this, dice, universe).await?; + let aquery_env = + get_aquery_env(ctx, &this.global_cfg_options_override).await?; aquery_env .deps( dice, @@ -244,18 +239,9 @@ fn aquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let aquery_env = get_aquery_env(ctx, this.target_platform.dupe()).await?; - - let targets = unpack_action_nodes( - targets, - &this.target_platform, - ctx, - dice, - aquery_env.as_ref(), - ) - .await?; + let targets = unpack_action_nodes(this, dice, targets).await?; - get_aquery_env(ctx, this.target_platform.dupe()) + get_aquery_env(ctx, &this.global_cfg_options_override) .await? .all_actions(dice, &targets) .await @@ -280,18 +266,9 @@ fn aquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let aquery_env = get_aquery_env(ctx, this.target_platform.dupe()).await?; - - let targets = unpack_action_nodes( - targets, - &this.target_platform, - ctx, - dice, - aquery_env.as_ref(), - ) - .await?; + let targets = unpack_action_nodes(this, dice, targets).await?; - get_aquery_env(ctx, this.target_platform.dupe()) + get_aquery_env(ctx, &this.global_cfg_options_override) .await? .all_outputs(dice, &targets) .await @@ -310,19 +287,10 @@ fn aquery_methods(builder: &mut MethodsBuilder) { value: &str, targets: UnpackActionNodes<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - let aquery_env = get_aquery_env(ctx, this.target_platform.dupe()).await?; - - let targets = unpack_action_nodes( - targets, - &this.target_platform, - ctx, - dice, - aquery_env.as_ref(), - ) - .await?; + let targets = unpack_action_nodes(this, dice, targets).await?; targets .attrfilter(attr, &|v| Ok(v == value)) @@ -365,7 +333,7 @@ fn aquery_methods(builder: &mut MethodsBuilder) { &ctx.working_dir()?, query, &query_args, - this.target_platform.dupe(), + this.global_cfg_options_override.clone(), ) .await?, eval, diff --git a/app/buck2_bxl/src/bxl/starlark_defs/artifacts.rs b/app/buck2_bxl/src/bxl/starlark_defs/artifacts.rs index 52baae6e49c32..d625f14906789 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/artifacts.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/artifacts.rs @@ -87,7 +87,7 @@ pub(crate) async fn visit_artifact_path_without_associated_deduped( if !visited.insert(ag.dupe()) { continue; } - match ag.resolved()? { + match ag.resolved_artifact(ctx).await? { ResolvedArtifactGroup::Artifact(a) => { visitor(a.get_path(), abs)?; } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/audit.rs b/app/buck2_bxl/src/bxl/starlark_defs/audit.rs index 9e9e5ee55d4b3..dba234fd4e134 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/audit.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/audit.rs @@ -12,9 +12,9 @@ use anyhow::Context as _; use buck2_build_api::audit_cell::audit_cell; use buck2_build_api::audit_output::audit_output; use buck2_build_api::audit_output::AuditOutputResult; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::CellResolver; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; -use buck2_core::target::label::TargetLabel; use buck2_interpreter::types::target_label::StarlarkTargetLabel; use derivative::Derivative; use derive_more::Display; @@ -63,7 +63,6 @@ pub(crate) struct StarlarkAuditCtx<'v> { #[trace(unsafe_ignore)] #[derivative(Debug = "ignore")] cell_resolver: CellResolver, - global_target_platform: Option, } #[starlark_value(type = "audit_ctx", StarlarkTypeRepr, UnpackValue)] @@ -85,13 +84,11 @@ impl<'v> StarlarkAuditCtx<'v> { ctx: &'v BxlContext<'v>, working_dir: ProjectRelativePathBuf, cell_resolver: CellResolver, - global_target_platform: Option, ) -> anyhow::Result { Ok(Self { ctx, working_dir, cell_resolver, - global_target_platform, }) } } @@ -124,10 +121,10 @@ fn audit_methods(builder: &mut MethodsBuilder) { heap: &'v Heap, ) -> anyhow::Result>> { let target_platform = target_platform.parse_target_platforms( - &this.ctx.data.target_alias_resolver, - &this.ctx.data.cell_resolver, - this.ctx.data.cell_name, - &this.global_target_platform, + this.ctx.target_alias_resolver(), + this.ctx.cell_resolver(), + this.ctx.cell_name(), + &this.ctx.global_cfg_options().target_platform, )?; this.ctx.async_ctx.borrow_mut().via(|ctx| { @@ -137,7 +134,10 @@ fn audit_methods(builder: &mut MethodsBuilder) { &this.working_dir, &this.cell_resolver, ctx, - target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, ) .await? .map(|result| { diff --git a/app/buck2_bxl/src/bxl/starlark_defs/cli_args.rs b/app/buck2_bxl/src/bxl/starlark_defs/cli_args.rs index 8c87d4d98807e..88c35e4462143 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/cli_args.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/cli_args.rs @@ -765,6 +765,31 @@ pub(crate) fn register_cli_args_module(registry: &mut GlobalsBuilder) { cli_args_module(registry) } +pub(crate) enum ArgAccessor<'a> { + Clap { + clap: &'a clap::ArgMatches, + arg: &'a str, + }, + Literal(&'a str), +} + +#[allow(deprecated)] // TODO(nga): fix. +impl<'a> ArgAccessor<'a> { + fn value_of(&self) -> Option<&str> { + match self { + ArgAccessor::Clap { clap, arg } => clap.value_of(arg), + ArgAccessor::Literal(s) => Some(s), + } + } + + fn values_of(&self) -> Option> { + match self { + ArgAccessor::Clap { clap, arg } => clap.values_of(arg).map(itertools::Either::Left), + ArgAccessor::Literal(s) => Some(itertools::Either::Right(std::iter::once(*s))), + } + } +} + #[cfg(test)] mod tests { use std::collections::HashSet; @@ -909,28 +934,3 @@ mod tests { Ok(()) } } - -pub(crate) enum ArgAccessor<'a> { - Clap { - clap: &'a clap::ArgMatches, - arg: &'a str, - }, - Literal(&'a str), -} - -#[allow(deprecated)] // TODO(nga): fix. -impl<'a> ArgAccessor<'a> { - fn value_of(&self) -> Option<&str> { - match self { - ArgAccessor::Clap { clap, arg } => clap.value_of(arg), - ArgAccessor::Literal(s) => Some(s), - } - } - - fn values_of(&self) -> Option> { - match self { - ArgAccessor::Clap { clap, arg } => clap.values_of(arg).map(itertools::Either::Left), - ArgAccessor::Literal(s) => Some(itertools::Either::Right(std::iter::once(*s))), - } - } -} diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context.rs b/app/buck2_bxl/src/bxl/starlark_defs/context.rs index ec356d44e8b97..dfeb60a5bfa6d 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context.rs @@ -8,13 +8,13 @@ */ //! The context containing the available buck commands and query operations for `bxl` functions. -//! use std::cell::RefCell; use std::cell::RefMut; use std::fmt::Display; use std::io::Write; use std::iter; +use std::ops::Deref; use std::rc::Rc; use std::sync::Arc; @@ -35,6 +35,8 @@ use buck2_cli_proto::build_request::Materializations; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::data::HasIoProvider; use buck2_common::events::HasEvents; +use buck2_common::global_cfg_options::GlobalCfgOptions; +use buck2_common::scope::scope_and_collect_with_dice; use buck2_common::target_aliases::BuckConfigTargetAliasResolver; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::base_deferred_key::BaseDeferredKeyDyn; @@ -54,7 +56,6 @@ use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersLabel; use buck2_core::target::label::TargetLabel; use buck2_events::dispatch::console_message; -use buck2_events::dispatch::with_dispatcher_async; use buck2_execute::digest_config::DigestConfig; use buck2_execute::digest_config::HasDigestConfig; use buck2_interpreter::dice::starlark_provider::with_starlark_eval_provider; @@ -218,23 +219,37 @@ pub(crate) struct BxlContext<'v> { pub(crate) data: BxlContextNoDice<'v>, } +impl<'v> Deref for BxlContext<'v> { + type Target = BxlContextNoDice<'v>; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + #[derive(Derivative, Display, Trace, NoSerialize, Allocative)] #[derivative(Debug)] #[display(fmt = "{:?}", self)] pub(crate) struct BxlContextNoDice<'v> { - pub(crate) current_bxl: BxlKey, - #[derivative(Debug = "ignore")] - pub(crate) target_alias_resolver: BuckConfigTargetAliasResolver, - pub(crate) cell_name: CellName, - pub(crate) cell_root_abs: AbsNormPathBuf, - #[derivative(Debug = "ignore")] - pub(crate) cell_resolver: CellResolver, pub(crate) state: ValueTyped<'v, AnalysisActions<'v>>, - pub(crate) global_target_platform: Option, pub(crate) context_type: BxlContextType<'v>, - pub(crate) project_fs: ProjectRoot, + core: BxlContextCoreData, +} + +#[derive(Derivative, Display, Trace, Allocative)] +#[derivative(Debug)] +#[display(fmt = "{:?}", self)] +pub(crate) struct BxlContextCoreData { + current_bxl: BxlKey, #[derivative(Debug = "ignore")] - pub(crate) artifact_fs: ArtifactFs, + target_alias_resolver: BuckConfigTargetAliasResolver, + cell_name: CellName, + cell_root_abs: AbsNormPathBuf, + #[derivative(Debug = "ignore")] + cell_resolver: CellResolver, + project_fs: ProjectRoot, + #[derivative(Debug = "ignore")] + artifact_fs: ArtifactFs, } impl<'v> BxlContext<'v> { @@ -251,7 +266,6 @@ impl<'v> BxlContext<'v> { output_sink: RefCell>, error_sink: RefCell>, digest_config: DigestConfig, - global_target_platform: Option, ) -> anyhow::Result { let cell_root_abs = project_fs.root().join( cell_resolver @@ -284,11 +298,6 @@ impl<'v> BxlContext<'v> { Ok(Self { async_ctx: async_ctx.clone(), data: BxlContextNoDice { - current_bxl, - target_alias_resolver, - cell_name, - cell_root_abs, - cell_resolver, state: heap.alloc_typed(AnalysisActions { state: RefCell::new(None), // TODO(nga): attributes struct should not be accessible to BXL. @@ -299,10 +308,16 @@ impl<'v> BxlContext<'v> { .into(), digest_config, }), - global_target_platform, context_type, - project_fs, - artifact_fs, + core: BxlContextCoreData { + current_bxl, + target_alias_resolver, + cell_name, + cell_root_abs, + cell_resolver, + project_fs, + artifact_fs, + }, }, }) } @@ -317,7 +332,6 @@ impl<'v> BxlContext<'v> { cell_name: CellName, async_ctx: Rc>>, digest_config: DigestConfig, - global_target_platform: Option, analysis_registry: AnalysisRegistry<'v>, dynamic_data: DynamicBxlContextData, ) -> anyhow::Result { @@ -331,11 +345,6 @@ impl<'v> BxlContext<'v> { Ok(Self { async_ctx, data: BxlContextNoDice { - current_bxl, - target_alias_resolver, - cell_name, - cell_root_abs, - cell_resolver, state: heap.alloc_typed(AnalysisActions { state: RefCell::new(Some(analysis_registry)), // TODO(nga): attributes struct should not be accessible to BXL. @@ -346,10 +355,16 @@ impl<'v> BxlContext<'v> { .into(), digest_config, }), - global_target_platform, context_type: BxlContextType::Dynamic(dynamic_data), - project_fs, - artifact_fs, + core: BxlContextCoreData { + current_bxl, + target_alias_resolver, + cell_name, + cell_root_abs, + cell_resolver, + project_fs, + artifact_fs, + }, }, }) } @@ -358,7 +373,7 @@ impl<'v> BxlContext<'v> { /// This should generally only be called at the top level functions in bxl. /// Within the lambdas, use the existing reference to Dice provided instead of calling nested /// via_dice, as that breaks borrow invariants of the dice computations. - pub fn via_dice<'a, 's, T>( + pub(crate) fn via_dice<'a, 's, T>( &'a self, f: impl for<'x> FnOnce( RefMut<'x, BxlSafeDiceComputations<'v>>, @@ -372,17 +387,6 @@ impl<'v> BxlContext<'v> { f(self.async_ctx.borrow_mut(), data) } - pub(crate) fn project_root(&self) -> &ProjectRoot { - self.data.project_root() - } - - /// Working dir for resolving literals. - /// Note, unlike buck2 command line UI, we resolve targets and literals - /// against the cell root instead of user working dir. - pub(crate) fn working_dir(&self) -> anyhow::Result { - self.data.working_dir() - } - /// Must take an `AnalysisContext` and `OutputStream` which has never had `take_state` called on it before. pub(crate) fn take_state( value: ValueTyped<'v, BxlContext<'v>>, @@ -448,29 +452,61 @@ impl<'v> BxlContextNoDice<'v> { } pub(crate) fn project_root(&self) -> &ProjectRoot { - &self.project_fs + &self.core.project_fs + } + + pub(crate) fn global_cfg_options(&self) -> &GlobalCfgOptions { + self.core.current_bxl.global_cfg_options() + } + + pub(crate) fn target_alias_resolver(&self) -> &BuckConfigTargetAliasResolver { + &self.core.target_alias_resolver + } + + pub(crate) fn cell_resolver(&self) -> &CellResolver { + &self.core.cell_resolver + } + + pub(crate) fn cell_name(&self) -> CellName { + self.core.cell_name + } + + pub(crate) fn cell_root_abs(&self) -> &AbsNormPathBuf { + &self.core.cell_root_abs + } + + pub(crate) fn current_bxl(&self) -> &BxlKey { + &self.core.current_bxl + } + + pub(crate) fn project_fs(&self) -> &ProjectRoot { + &self.core.project_fs + } + + pub(crate) fn artifact_fs(&self) -> &ArtifactFs { + &self.core.artifact_fs } /// Working dir for resolving literals. /// Note, unlike buck2 command line UI, we resolve targets and literals /// against the cell root instead of user working dir. pub(crate) fn working_dir(&self) -> anyhow::Result { - let cell = self.cell_resolver.get(self.cell_name)?; + let cell = self.cell_resolver().get(self.cell_name())?; Ok(cell.path().as_project_relative_path().to_owned()) } pub(crate) fn parse_query_file_literal(&self, literal: &str) -> anyhow::Result { parse_query_file_literal( literal, - self.cell_resolver - .get(self.cell_name)? + self.cell_resolver() + .get(self.cell_name())? .cell_alias_resolver(), - &self.cell_resolver, + self.cell_resolver(), // NOTE(nga): we pass cell root as working directory here, // which is inconsistent with the rest of buck2: // The same query `owner(foo.h)` is resolved using // current directory in `buck2 query`, but relative to cell root in BXL. - &self.cell_root_abs, + self.cell_root_abs(), self.project_root(), ) } @@ -492,7 +528,6 @@ pub(crate) async fn eval_bxl_for_dynamic_output<'v>( exec_deps: dynamic_key.0.exec_deps.clone(), toolchains: dynamic_key.0.toolchains.clone(), }; - let global_target_platform = key.global_target_platform().dupe(); let label = key.label(); let cell_resolver = dice_ctx.get_cell_resolver().await?; let cell = label.bxl_path.cell(); @@ -526,9 +561,9 @@ pub(crate) async fn eval_bxl_for_dynamic_output<'v>( // on the scope will be dropped at the earliest await point. If we are within the blocking // section of bxl, the cancellation observer will be notified and cause the blocking calls // to terminate. - async_scoped::TokioScope::scope_and_collect(|s| { + scope_and_collect_with_dice(dice_ctx, |dice_ctx, s| { s.spawn_cancellable( - with_dispatcher_async(dispatcher.dupe(), async move { + async move { with_starlark_eval_provider( dice_ctx, &mut StarlarkProfilerOrInstrumentation::disabled(), @@ -559,7 +594,6 @@ pub(crate) async fn eval_bxl_for_dynamic_output<'v>( cell_name, async_ctx, digest_config, - global_target_platform, dynamic_lambda_ctx_data.registry, dynamic_data, )?; @@ -601,7 +635,7 @@ pub(crate) async fn eval_bxl_for_dynamic_output<'v>( }, ) .await - }), + }, || Err(anyhow::anyhow!("cancelled")), ) }) @@ -693,7 +727,7 @@ fn context_methods(builder: &mut MethodsBuilder) { .context_type .unpack_root() .context(BxlContextDynamicError::Unsupported("root".to_owned()))?; - Ok(this.data.cell_root_abs.to_owned().to_string()) + Ok(this.cell_root_abs().to_owned().to_string()) } /// Gets the target nodes for the `labels`, accepting an optional `target_platform` which is the @@ -723,10 +757,10 @@ fn context_methods(builder: &mut MethodsBuilder) { Either, StarlarkTargetSet>, > { let target_platform = target_platform.parse_target_platforms( - &this.data.target_alias_resolver, - &this.data.cell_resolver, - this.data.cell_name, - &this.data.global_target_platform, + this.target_alias_resolver(), + this.cell_resolver(), + this.cell_name(), + &this.global_cfg_options().target_platform, )?; this.via_dice(|mut dice, this| { @@ -735,7 +769,10 @@ fn context_methods(builder: &mut MethodsBuilder) { let target_expr = TargetListExpr::<'v, ConfiguredTargetNode>::unpack_allow_unconfigured( labels, - &target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, this, ctx, ) @@ -847,10 +884,10 @@ fn context_methods(builder: &mut MethodsBuilder) { #[starlark(require = named, default = false)] keep_going: bool, ) -> anyhow::Result> { let target_platform = target_platform.parse_target_platforms( - &this.data.target_alias_resolver, - &this.data.cell_resolver, - this.data.cell_name, - &this.data.global_target_platform, + this.target_alias_resolver(), + this.cell_resolver(), + this.cell_name(), + &this.global_cfg_options().target_platform, )?; this.via_dice(|mut ctx, this_no_dice: &BxlContextNoDice<'_>| { @@ -859,7 +896,10 @@ fn context_methods(builder: &mut MethodsBuilder) { let target_expr = if keep_going { TargetListExpr::<'v, ConfiguredTargetNode>::unpack_keep_going( labels, - &target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, this_no_dice, ctx, ) @@ -867,7 +907,10 @@ fn context_methods(builder: &mut MethodsBuilder) { } else { TargetListExpr::<'v, ConfiguredTargetNode>::unpack_allow_unconfigured( labels, - &target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, this_no_dice, ctx, ) @@ -902,7 +945,7 @@ fn context_methods(builder: &mut MethodsBuilder) { #[starlark(default = ValueAsStarlarkTargetLabel::NONE)] target_platform: ValueAsStarlarkTargetLabel<'v>, ) -> anyhow::Result> { - StarlarkCQueryCtx::new(this, target_platform, &this.data.global_target_platform) + StarlarkCQueryCtx::new(this, target_platform, this.data.global_cfg_options()) } /// Returns the `aqueryctx` that holds all the aquery functions. @@ -915,7 +958,11 @@ fn context_methods(builder: &mut MethodsBuilder) { #[starlark(default = ValueAsStarlarkTargetLabel::NONE)] target_platform: ValueAsStarlarkTargetLabel<'v>, ) -> anyhow::Result> { - StarlarkAQueryCtx::new(this, target_platform, &this.data.global_target_platform) + StarlarkAQueryCtx::new( + this, + target_platform, + &this.data.global_cfg_options().target_platform, + ) } /// Returns the bxl actions to create and register actions for this @@ -991,10 +1038,10 @@ fn context_methods(builder: &mut MethodsBuilder) { let (exec_deps, toolchains) = match &this.context_type { BxlContextType::Root { .. } => { let target_platform = target_platform.parse_target_platforms( - &this.target_alias_resolver, - &this.cell_resolver, - this.cell_name, - &this.global_target_platform, + this.target_alias_resolver(), + this.cell_resolver(), + this.cell_name(), + &this.global_cfg_options().target_platform, )?; let exec_deps = match exec_deps { NoneOr::None => Vec::new(), @@ -1016,8 +1063,8 @@ fn context_methods(builder: &mut MethodsBuilder) { } }; - let exec_compatible_with = match exec_compatible_with { - NoneOr::None => Vec::new(), + let exec_compatible_with: Arc<[_]> = match exec_compatible_with { + NoneOr::None => Arc::new([]), NoneOr::Other(exec_compatible_with) => { TargetListExpr::::unpack( exec_compatible_with, @@ -1035,7 +1082,7 @@ fn context_methods(builder: &mut MethodsBuilder) { let execution_resolution = resolve_bxl_execution_platform( ctx, - this.cell_name, + this.cell_name(), exec_deps, toolchains, target_platform.clone(), @@ -1114,10 +1161,10 @@ fn context_methods(builder: &mut MethodsBuilder) { >, > { let target_platform = target_platform.parse_target_platforms( - &this.data.target_alias_resolver, - &this.data.cell_resolver, - this.data.cell_name, - &this.data.global_target_platform, + this.data.target_alias_resolver(), + this.data.cell_resolver(), + this.data.cell_name(), + &this.data.global_cfg_options().target_platform, )?; let res: anyhow::Result<_> = this.via_dice(|mut dice, ctx| { @@ -1125,7 +1172,10 @@ fn context_methods(builder: &mut MethodsBuilder) { async { let providers = ProvidersExpr::::unpack( labels, - target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, ctx, dice, ) @@ -1244,10 +1294,10 @@ fn context_methods(builder: &mut MethodsBuilder) { ctx.via(|ctx| { async move { match ParsedPattern::::parse_relaxed( - &this_no_dice.target_alias_resolver, - CellPathRef::new(this_no_dice.cell_name, CellRelativePath::empty()), + this_no_dice.target_alias_resolver(), + CellPathRef::new(this_no_dice.cell_name(), CellRelativePath::empty()), label, - &this_no_dice.cell_resolver, + this_no_dice.cell_resolver(), )? { ParsedPattern::Target(pkg, name, TargetPatternExtra) => { let target_label = TargetLabel::new(pkg, name.as_ref()); @@ -1267,8 +1317,8 @@ fn context_methods(builder: &mut MethodsBuilder) { ctx.via(|ctx| { async move { Ok(( - this.cell_resolver - .get(this.cell_name)? + this.cell_resolver() + .get(this.cell_name())? .path() .as_project_relative_path() .to_buf(), @@ -1279,12 +1329,7 @@ fn context_methods(builder: &mut MethodsBuilder) { }) })?; - StarlarkAuditCtx::new( - this, - working_dir, - cell_resolver, - this.data.global_target_platform.clone(), - ) + StarlarkAuditCtx::new(this, working_dir, cell_resolver) } /// Awaits a promise and returns an optional value of the promise. @@ -1316,7 +1361,7 @@ fn context_methods(builder: &mut MethodsBuilder) { this.via_dice(|mut dice, this| { dice.via(|dice| { action_factory - .run_promises(dice, eval, format!("bxl$promises:{}", &this.current_bxl)) + .run_promises(dice, eval, format!("bxl$promises:{}", this.current_bxl())) .boxed_local() }) })?; @@ -1335,8 +1380,8 @@ fn context_methods(builder: &mut MethodsBuilder) { #[starlark(require = named)] metadata: Value<'v>, ) -> anyhow::Result { let parser = StarlarkUserEventParser { - artifact_fs: &this.data.artifact_fs, - project_fs: &this.data.project_fs, + artifact_fs: this.artifact_fs(), + project_fs: this.project_fs(), }; let event = parser.parse(id, metadata)?; diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context/actions.rs b/app/buck2_bxl/src/bxl/starlark_defs/context/actions.rs index 7aca468576423..66e46b7392a93 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context/actions.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context/actions.rs @@ -8,7 +8,8 @@ */ //! Starlark Actions API for bxl functions -//! +use std::sync::Arc; + use allocative::Allocative; use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; use buck2_build_api::analysis::registry::AnalysisRegistry; @@ -16,6 +17,7 @@ use buck2_build_api::interpreter::rule_defs::context::AnalysisActions; use buck2_build_api::interpreter::rule_defs::provider::dependency::Dependency; use buck2_configured::configuration::calculation::ConfigurationCalculation; use buck2_configured::nodes::calculation::ExecutionPlatformConstraints; +use buck2_configured::target::TargetConfiguredTargetLabel; use buck2_core::base_deferred_key::BaseDeferredKey; use buck2_core::cells::name::CellName; use buck2_core::configuration::data::ConfigurationData; @@ -72,7 +74,7 @@ pub(crate) async fn resolve_bxl_execution_platform( exec_deps: Vec, toolchain_deps: Vec, target_platform: Option, - exec_compatible_with: Vec, + exec_compatible_with: Arc<[TargetLabel]>, module: &Module, ) -> anyhow::Result { // bxl has on transitions @@ -86,7 +88,7 @@ pub(crate) async fn resolve_bxl_execution_platform( None => ConfigurationData::unspecified(), }; let resolved_configuration = { - ctx.get_resolved_configuration(&platform_configuration, cell, &exec_compatible_with) + ctx.get_resolved_configuration(&platform_configuration, cell, &*exec_compatible_with) .await? }; @@ -116,7 +118,7 @@ pub(crate) async fn resolve_bxl_execution_platform( .collect(), toolchain_deps_configured .iter() - .map(|dep| dep.target().clone()) + .map(|dep| TargetConfiguredTargetLabel::new_without_exec_cfg(dep.target().dupe())) .collect(), exec_compatible_with, ); @@ -175,7 +177,7 @@ pub(crate) fn validate_action_instantiation( } else { let execution_platform = bxl_execution_resolution.resolved_execution.clone(); let analysis_registry = AnalysisRegistry::new_from_owner( - BaseDeferredKey::BxlLabel(this.current_bxl.dupe().into_base_deferred_key_dyn_impl( + BaseDeferredKey::BxlLabel(this.current_bxl().dupe().into_base_deferred_key_dyn_impl( execution_platform.clone(), bxl_execution_resolution.exec_deps_configured.clone(), bxl_execution_resolution.toolchain_deps_configured.clone(), diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context/build.rs b/app/buck2_bxl/src/bxl/starlark_defs/context/build.rs index e2b63e9e54245..03c0a00b36a6d 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context/build.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context/build.rs @@ -24,6 +24,7 @@ use buck2_build_api::build::ProvidersToBuild; use buck2_build_api::bxl::build_result::BxlBuildResult; use buck2_build_api::interpreter::rule_defs::artifact::StarlarkArtifact; use buck2_cli_proto::build_request::Materializations; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use dashmap::DashMap; @@ -85,6 +86,7 @@ where .0 .unpack_built() .unwrap() + .1 .outputs .iter() .filter_map(|built| built.as_ref().ok()) @@ -146,6 +148,7 @@ where .0 .unpack_built() .unwrap() + .1 .outputs .iter() .filter_map(|built| built.as_ref().err()) @@ -193,10 +196,10 @@ pub(crate) fn build<'v>( ConvertMaterializationContext::with_existing_map(materializations, materializations_map); let target_platform = target_platform.parse_target_platforms( - &ctx.data.target_alias_resolver, - &ctx.data.cell_resolver, - ctx.data.cell_name, - &ctx.data.global_target_platform, + ctx.target_alias_resolver(), + ctx.cell_resolver(), + ctx.cell_name(), + &ctx.data.global_cfg_options().target_platform, )?; let build_result = ctx.via_dice( @@ -205,7 +208,10 @@ pub(crate) fn build<'v>( async { let build_spec = ProvidersExpr::::unpack( spec, - target_platform, + &GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into() + }, ctx, dice, ) @@ -261,11 +267,14 @@ pub(crate) fn build<'v>( .map(|(target, result)| { ( eval.heap() - .alloc_typed(StarlarkConfiguredProvidersLabel::new(target)) + .alloc_typed(StarlarkConfiguredProvidersLabel::new(target.clone())) .hashed() .unwrap(), eval.heap() - .alloc_typed(StarlarkBxlBuildResult(BxlBuildResult::new(result))), + .alloc_typed(StarlarkBxlBuildResult(BxlBuildResult::new( + target.clone(), + result, + ))), ) }) .collect()) diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context/fs.rs b/app/buck2_bxl/src/bxl/starlark_defs/context/fs.rs index 75c6e54cd8eb0..004214a75476c 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context/fs.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context/fs.rs @@ -83,15 +83,15 @@ impl<'v> BxlFilesystem<'v> { } fn artifact_fs(&self) -> &ArtifactFs { - &self.ctx.data.artifact_fs + self.ctx.artifact_fs() } fn project_fs(&self) -> &ProjectRoot { - &self.ctx.data.project_fs + self.ctx.project_fs() } fn cell(&self) -> anyhow::Result<&CellInstance> { - self.ctx.data.cell_resolver.get(self.ctx.data.cell_name) + self.ctx.cell_resolver().get(self.ctx.cell_name()) } } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context/output.rs b/app/buck2_bxl/src/bxl/starlark_defs/context/output.rs index d9e91199cac6e..b4126a37af3de 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context/output.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context/output.rs @@ -494,7 +494,7 @@ fn get_artifacts_from_bxl_build_result( ) -> anyhow::Result> { match &bxl_build_result.0 { BxlBuildResult::None => Ok(Vec::new()), - BxlBuildResult::Built(result) => result + BxlBuildResult::Built { result, .. } => result .outputs .iter() .filter_map(|built| { diff --git a/app/buck2_bxl/src/bxl/starlark_defs/context/starlark_async.rs b/app/buck2_bxl/src/bxl/starlark_defs/context/starlark_async.rs index 8e49ddc8b8d27..445bfa7f74e82 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/context/starlark_async.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/context/starlark_async.rs @@ -35,7 +35,10 @@ enum ViaError { /// This is not exposed to starlark but rather, used by operations exposed to starlark to run /// code. /// This also provides a handle for dice. -pub struct BxlSafeDiceComputations<'a>(pub(super) &'a mut DiceComputations, CancellationObserver); +pub(crate) struct BxlSafeDiceComputations<'a>( + pub(super) &'a mut DiceComputations, + CancellationObserver, +); /// For a `via_dice`, the DiceComputations provided to each lambda is a reference that's only /// available for some specific lifetime `'x`. This is express as a higher rank lifetime bound @@ -44,7 +47,7 @@ pub struct BxlSafeDiceComputations<'a>(pub(super) &'a mut DiceComputations, Canc /// here which forces rust compiler to infer additional bounds on the `for <'x>` as a /// `&'x DiceComputationRef<'a>` cannot live more than `'a`, so using this type as the argument /// to the closure forces the correct lifetime bounds to be inferred by rust. -pub struct DiceComputationsRef<'s>(&'s mut DiceComputations); +pub(crate) struct DiceComputationsRef<'s>(&'s mut DiceComputations); impl<'s> Deref for DiceComputationsRef<'s> { type Target = DiceComputations; @@ -61,12 +64,12 @@ impl<'s> DerefMut for DiceComputationsRef<'s> { } impl<'a> BxlSafeDiceComputations<'a> { - pub fn new(dice: &'a mut DiceComputations, cancellation: CancellationObserver) -> Self { + pub(crate) fn new(dice: &'a mut DiceComputations, cancellation: CancellationObserver) -> Self { Self(dice, cancellation) } /// runs any async computation - pub fn via<'s, T>( + pub(crate) fn via<'s, T>( &'s mut self, f: impl for<'x> FnOnce(&'x mut DiceComputationsRef<'s>) -> LocalBoxFuture<'x, anyhow::Result>, ) -> anyhow::Result @@ -97,11 +100,11 @@ impl<'a> BxlSafeDiceComputations<'a> { }) } - pub fn global_data(&self) -> &DiceData { + pub(crate) fn global_data(&self) -> &DiceData { self.0.global_data() } - pub fn per_transaction_data(&self) -> &UserComputationData { + pub(crate) fn per_transaction_data(&self) -> &UserComputationData { self.0.per_transaction_data() } } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/cquery.rs b/app/buck2_bxl/src/bxl/starlark_defs/cquery.rs index 54bacf9c40296..25e2852c47e25 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/cquery.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/cquery.rs @@ -12,12 +12,13 @@ use buck2_build_api::query::bxl::BxlCqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_CQUERY_FUNCTIONS; use buck2_build_api::query::oneshot::CqueryOwnerBehavior; use buck2_build_api::query::oneshot::QUERY_FRONTEND; -use buck2_core::target::label::TargetLabel; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_node::nodes::configured::ConfiguredTargetNode; -use buck2_query::query::syntax::simple::eval::set::TargetSetExt; +use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::syntax::simple::functions::helpers::CapturedExpr; use derivative::Derivative; use derive_more::Display; +use dice::DiceComputations; use dupe::Dupe; use futures::FutureExt; use gazebo::prelude::*; @@ -68,7 +69,8 @@ pub(crate) struct StarlarkCQueryCtx<'v> { #[derivative(Debug = "ignore")] ctx: &'v BxlContext<'v>, #[derivative(Debug = "ignore")] - target_platform: Option, + // Overrides the GlobalCfgOptions in the BxlContext + global_cfg_options_override: GlobalCfgOptions, } #[starlark_value(type = "cqueryctx", StarlarkTypeRepr, UnpackValue)] @@ -87,33 +89,56 @@ impl<'v> AllocValue<'v> for StarlarkCQueryCtx<'v> { pub(crate) async fn get_cquery_env( ctx: &BxlContextNoDice<'_>, - target_platform: Option, + global_cfg_options_override: &GlobalCfgOptions, ) -> anyhow::Result> { (NEW_BXL_CQUERY_FUNCTIONS.get()?)( - target_platform, + global_cfg_options_override.clone(), ctx.project_root().dupe(), - ctx.cell_name, - ctx.cell_resolver.dupe(), + ctx.cell_name(), + ctx.cell_resolver().dupe(), ) .await } +async fn unpack_targets<'v>( + this: &StarlarkCQueryCtx<'v>, + dice: &mut DiceComputations, + targets: ConfiguredTargetListExprArg<'v>, +) -> anyhow::Result> { + filter_incompatible( + TargetListExpr::<'v, ConfiguredTargetNode>::unpack( + targets, + &this.global_cfg_options_override, + &this.ctx.data, + dice, + ) + .await? + .get(dice) + .await? + .into_iter(), + &this.ctx.data, + ) +} + impl<'v> StarlarkCQueryCtx<'v> { pub(crate) fn new( ctx: &'v BxlContext<'v>, global_target_platform: ValueAsStarlarkTargetLabel<'v>, - default_target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result> { let target_platform = global_target_platform.parse_target_platforms( - &ctx.data.target_alias_resolver, - &ctx.data.cell_resolver, - ctx.data.cell_name, - default_target_platform, + ctx.target_alias_resolver(), + ctx.cell_resolver(), + ctx.cell_name(), + &global_cfg_options.target_platform, )?; Ok(Self { ctx, - target_platform, + global_cfg_options_override: GlobalCfgOptions { + target_platform, + cli_modifiers: vec![].into(), + }, }) } } @@ -134,33 +159,9 @@ fn cquery_methods(builder: &mut MethodsBuilder) { this.ctx.via_dice(move |mut dice, ctx| { dice.via(|dice| { async move { - let from = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - from, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - let to = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - to, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - get_cquery_env(ctx, this.target_platform.dupe()) + let from = unpack_targets(this, dice, from).await?; + let to = unpack_targets(this, dice, to).await?; + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .allpaths(dice, &from, &to) .await @@ -180,33 +181,9 @@ fn cquery_methods(builder: &mut MethodsBuilder) { this.ctx.via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let from = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - from, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - let to = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - to, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - get_cquery_env(ctx, this.target_platform.dupe()) + let from = unpack_targets(this, dice, from).await?; + let to = unpack_targets(this, dice, to).await?; + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .somepath(dice, &from, &to) .await @@ -224,24 +201,13 @@ fn cquery_methods(builder: &mut MethodsBuilder) { value: &str, targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) + unpack_targets(this, dice, targets) .await? - .get(dice) - .await? - .into_iter(), - ctx, - )? - .attrfilter(attr, &|v| Ok(v == value)) - .map(StarlarkTargetSet::from) + .attrfilter(attr, &|v| Ok(v == value)) + .map(StarlarkTargetSet::from) } .boxed_local() }) @@ -261,24 +227,13 @@ fn cquery_methods(builder: &mut MethodsBuilder) { regex: &str, targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) + unpack_targets(this, dice, targets) .await? - .into_iter(), - ctx, - )? - .kind(regex) - .map(StarlarkTargetSet::from) + .kind(regex) + .map(StarlarkTargetSet::from) } .boxed_local() }) @@ -299,24 +254,13 @@ fn cquery_methods(builder: &mut MethodsBuilder) { value: &str, targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) + unpack_targets(this, dice, targets) .await? - .get(dice) - .await? - .into_iter(), - ctx, - )? - .attrregexfilter(attribute, value) - .map(StarlarkTargetSet::from) + .attrregexfilter(attribute, value) + .map(StarlarkTargetSet::from) } .boxed_local() }) @@ -344,23 +288,11 @@ fn cquery_methods(builder: &mut MethodsBuilder) { dice.via(|dice| { async { let universe = match universe.into_option() { - Some(universe) => Some(filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - universe, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?), + Some(universe) => Some(unpack_targets(this, dice, universe).await?), None => None, }; - get_cquery_env(ctx, this.target_platform.dupe()) + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .owner(dice, files.get(ctx).await?.as_ref(), universe.as_ref()) .await @@ -393,21 +325,9 @@ fn cquery_methods(builder: &mut MethodsBuilder) { .into_option() .try_map(buck2_query_parser::parse_expr)?; - let targets = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - universe, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; + let targets = unpack_targets(this, dice, universe).await?; - get_cquery_env(ctx, this.target_platform.dupe()) + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .deps( dice, @@ -440,23 +360,12 @@ fn cquery_methods(builder: &mut MethodsBuilder) { targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result> { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { async { - filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) + unpack_targets(this, dice, targets) .await? - .into_iter(), - ctx, - )? - .filter_name(regex) + .filter_name(regex) } .boxed_local() }) @@ -477,25 +386,9 @@ fn cquery_methods(builder: &mut MethodsBuilder) { targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { - async { - filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )? - .inputs() - } - .boxed_local() + async { unpack_targets(this, dice, targets).await?.inputs() }.boxed_local() }) }) .map(StarlarkFileSet::from) @@ -510,20 +403,8 @@ fn cquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let targets = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - get_cquery_env(ctx, this.target_platform.dupe()) + let targets = unpack_targets(this, dice, targets).await?; + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .testsof(dice, &targets) .await @@ -544,23 +425,12 @@ fn cquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let targets = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack_allow_unconfigured( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - let maybe_compatibles = get_cquery_env(ctx, this.target_platform.dupe()) - .await? - .testsof_with_default_target_platform(dice, &targets) - .await?; + let targets = unpack_targets(this, dice, targets).await?; + let maybe_compatibles = + get_cquery_env(ctx, &this.global_cfg_options_override) + .await? + .testsof_with_default_target_platform(dice, &targets) + .await?; filter_incompatible(maybe_compatibles.into_iter(), ctx) } @@ -588,33 +458,9 @@ fn cquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let universe = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - universe, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - let targets = filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - from, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - get_cquery_env(ctx, this.target_platform.dupe()) + let universe = unpack_targets(this, dice, universe).await?; + let targets = unpack_targets(this, dice, from).await?; + get_cquery_env(ctx, &this.global_cfg_options_override) .await? .rdeps(dice, &universe, &targets, depth) .await @@ -662,7 +508,7 @@ fn cquery_methods(builder: &mut MethodsBuilder) { CqueryOwnerBehavior::Correct, query, &query_args, - this.target_platform.dupe(), + this.global_cfg_options_override.clone(), target_universe.into_option().as_ref().map(|v| &v.items[..]), ) .await?, @@ -688,23 +534,10 @@ fn cquery_methods(builder: &mut MethodsBuilder) { targets: ConfiguredTargetListExprArg<'v>, ) -> anyhow::Result { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = &filter_incompatible( - TargetListExpr::<'v, ConfiguredTargetNode>::unpack( - targets, - &this.target_platform, - ctx, - dice, - ) - .await? - .get(dice) - .await? - .into_iter(), - ctx, - )?; - + let targets = unpack_targets(this, dice, targets).await?; Ok(targets.buildfile()) } .boxed_local() diff --git a/app/buck2_bxl/src/bxl/starlark_defs/event.rs b/app/buck2_bxl/src/bxl/starlark_defs/event.rs index 00fc10281c02c..8845f4ef34290 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/event.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/event.rs @@ -8,7 +8,6 @@ */ //! Parse some inputs to a `[`StarlarkUserEvent`]. -//! use std::collections::HashMap; diff --git a/app/buck2_bxl/src/bxl/starlark_defs/functions.rs b/app/buck2_bxl/src/bxl/starlark_defs/functions.rs index a2f4896672bea..bcae0aa1a6efd 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/functions.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/functions.rs @@ -98,15 +98,15 @@ pub(crate) fn register_artifact_function(builder: &mut GlobalsBuilder) { get_artifact_path_display( artifact.get_path(), abs, - &ctx.data.project_fs, - &ctx.data.artifact_fs, + ctx.project_fs(), + ctx.artifact_fs(), )? } ValueAsArtifactLikeUnpack::DeclaredArtifact(a) => get_artifact_path_display( a.get_artifact_path(), abs, - &ctx.data.project_fs, - &ctx.data.artifact_fs, + ctx.project_fs(), + ctx.artifact_fs(), )?, _ => return Err(PromiseArtifactsNotSupported.into()), }; @@ -156,8 +156,8 @@ pub(crate) fn register_artifact_function(builder: &mut GlobalsBuilder) { let path = get_artifact_path_display( artifact_path, abs, - &bxl_ctx.project_fs, - &bxl_ctx.artifact_fs, + bxl_ctx.project_fs(), + bxl_ctx.artifact_fs(), )?; paths.push(path); diff --git a/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured.rs b/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured.rs index 966755c6fda5a..c83bc83648a46 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured.rs @@ -253,7 +253,7 @@ fn configured_target_node_value_methods(builder: &mut MethodsBuilder) { ctx: &'v BxlContext<'v>, eval: &mut Evaluator<'v, '_>, ) -> anyhow::Result> { - let configured_node = &this.0; + let configured_node = this.0.as_ref(); let dep_analysis: anyhow::Result, _> = ctx .async_ctx diff --git a/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured/attr_resolution_ctx.rs b/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured/attr_resolution_ctx.rs index d18cfa1dc2427..c16cf0267f499 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured/attr_resolution_ctx.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/nodes/configured/attr_resolution_ctx.rs @@ -46,11 +46,9 @@ impl<'v> LazyAttrResolutionContext<'v> { &self, ) -> &anyhow::Result> { self.dep_analysis_results.get_or_init(|| { - get_deps_from_analysis_results( - self.ctx.async_ctx.borrow_mut().via(|dice_ctx| { - get_dep_analysis(self.configured_node, dice_ctx).boxed_local() - })?, - ) + get_deps_from_analysis_results(self.ctx.async_ctx.borrow_mut().via(|dice_ctx| { + get_dep_analysis(self.configured_node.as_ref(), dice_ctx).boxed_local() + })?) }) } @@ -58,10 +56,9 @@ impl<'v> LazyAttrResolutionContext<'v> { &self, ) -> &anyhow::Result>> { self.query_results.get_or_init(|| { - self.ctx - .async_ctx - .borrow_mut() - .via(|dice_ctx| resolve_queries(dice_ctx, self.configured_node).boxed_local()) + self.ctx.async_ctx.borrow_mut().via(|dice_ctx| { + resolve_queries(dice_ctx, self.configured_node.as_ref()).boxed_local() + }) }) } } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/nodes/unconfigured/attribute.rs b/app/buck2_bxl/src/bxl/starlark_defs/nodes/unconfigured/attribute.rs index 7cbc6533b86fa..ec355efa19fac 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/nodes/unconfigured/attribute.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/nodes/unconfigured/attribute.rs @@ -123,7 +123,7 @@ fn coerced_attr_methods(builder: &mut MethodsBuilder) { } } -pub trait CoercedAttrExt { +pub(crate) trait CoercedAttrExt { fn starlark_type(&self) -> anyhow::Result<&'static str>; fn to_value<'v>(&self, pkg: PackageLabel, heap: &'v Heap) -> anyhow::Result>; diff --git a/app/buck2_bxl/src/bxl/starlark_defs/providers_expr.rs b/app/buck2_bxl/src/bxl/starlark_defs/providers_expr.rs index 0b36c95aed320..d59232036c8fe 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/providers_expr.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/providers_expr.rs @@ -7,6 +7,7 @@ * of this source tree. */ +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::cell_path::CellPathRef; use buck2_core::cells::paths::CellRelativePath; use buck2_core::pattern::pattern_type::ProvidersPatternExtra; @@ -15,7 +16,6 @@ use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersLabel; use buck2_core::provider::label::ProvidersLabelMaybeConfigured; use buck2_core::provider::label::ProvidersName; -use buck2_core::target::label::TargetLabel; use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use buck2_interpreter::types::configured_providers_label::StarlarkProvidersLabel; use buck2_interpreter::types::target_label::StarlarkConfiguredTargetLabel; @@ -88,23 +88,23 @@ pub(crate) enum ConfiguredProvidersExprArg<'v> { impl ProvidersExpr { pub(crate) async fn unpack<'v, 'c>( arg: ConfiguredProvidersExprArg<'v>, - target_platform: Option, + global_cfg_options_override: &GlobalCfgOptions, ctx: &BxlContextNoDice<'_>, dice: &'c DiceComputations, ) -> anyhow::Result { match arg { ConfiguredProvidersExprArg::One(arg) => Ok(ProvidersExpr::Literal( - Self::unpack_literal(arg, &target_platform, ctx, dice).await?, + Self::unpack_literal(arg, global_cfg_options_override, ctx, dice).await?, )), ConfiguredProvidersExprArg::List(arg) => { - Ok(Self::unpack_iterable(arg, &target_platform, ctx, dice).await?) + Ok(Self::unpack_iterable(arg, global_cfg_options_override, ctx, dice).await?) } } } async fn unpack_literal<'v, 'c>( arg: ConfiguredProvidersLabelArg<'v>, - target_platform: &'c Option, + global_cfg_options_override: &'c GlobalCfgOptions, ctx: &BxlContextNoDice<'_>, dice: &'c DiceComputations, ) -> anyhow::Result { @@ -126,7 +126,7 @@ impl ProvidersExpr { } ConfiguredProvidersLabelArg::Unconfigured(arg) => { let label = Self::unpack_providers_label(arg, ctx)?; - dice.get_configured_provider_label(&label, target_platform.as_ref()) + dice.get_configured_provider_label(&label, global_cfg_options_override) .await } } @@ -134,7 +134,7 @@ impl ProvidersExpr { async fn unpack_iterable<'c, 'v: 'c>( arg: ConfiguredProvidersLabelListArg<'v>, - target_platform: &'c Option, + global_cfg_options_override: &'c GlobalCfgOptions, ctx: &'c BxlContextNoDice<'_>, dice: &'c DiceComputations, ) -> anyhow::Result> { @@ -142,8 +142,11 @@ impl ProvidersExpr { ConfiguredProvidersLabelListArg::StarlarkTargetSet(s) => Ok(ProvidersExpr::Iterable( future::try_join_all(s.0.iter().map(|node| async { let providers_label = ProvidersLabel::default_for(node.label().dupe()); - dice.get_configured_provider_label(&providers_label, target_platform.as_ref()) - .await + dice.get_configured_provider_label( + &providers_label, + global_cfg_options_override, + ) + .await })) .await?, )), @@ -157,7 +160,9 @@ impl ProvidersExpr { ConfiguredProvidersLabelListArg::List(iterable) => { let mut res = Vec::new(); for arg in iterable.items { - res.push(Self::unpack_literal(arg, target_platform, ctx, dice).await?); + res.push( + Self::unpack_literal(arg, global_cfg_options_override, ctx, dice).await?, + ); } Ok(Self::Iterable(res)) @@ -220,11 +225,11 @@ impl ProvidersExpr

{ match arg { ProvidersLabelArg::Str(s) => { Ok(ParsedPattern::::parse_relaxed( - &ctx.target_alias_resolver, + ctx.target_alias_resolver(), // TODO(nga): Parse relaxed relative to cell root is incorrect. - CellPathRef::new(ctx.cell_name, CellRelativePath::empty()), + CellPathRef::new(ctx.cell_name(), CellRelativePath::empty()), s, - &ctx.cell_resolver, + ctx.cell_resolver(), )? .as_providers_label(s)?) } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/target_expr.rs b/app/buck2_bxl/src/bxl/starlark_defs/target_expr.rs index 5a943f1ca4c5c..16560b44ea9a1 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/target_expr.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/target_expr.rs @@ -18,13 +18,13 @@ use dupe::Dupe; #[derive(Clone)] pub(crate) enum TargetExpr<'v, Node: QueryTarget> { Node(Node), - Label(Cow<'v, Node::NodeRef>), + Label(Cow<'v, Node::Key>), } impl<'v, Node: QueryTarget> TargetExpr<'v, Node> { - pub(crate) fn node_ref(&self) -> &Node::NodeRef { + pub(crate) fn node_ref(&self) -> &Node::Key { match self { - TargetExpr::Node(node) => node.node_ref(), + TargetExpr::Node(node) => node.node_key(), TargetExpr::Label(label) => label, } } diff --git a/app/buck2_bxl/src/bxl/starlark_defs/target_list_expr.rs b/app/buck2_bxl/src/bxl/starlark_defs/target_list_expr.rs index dd9f9987fe1b8..ff53db4365d70 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/target_list_expr.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/target_list_expr.rs @@ -12,6 +12,7 @@ use std::iter; use anyhow::Context; use buck2_build_api::configure_targets::get_maybe_compatible_targets; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::cell_path::CellPathRef; use buck2_core::cells::paths::CellRelativePath; use buck2_core::configuration::compatibility::IncompatiblePlatformReason; @@ -249,7 +250,7 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { pub(crate) async fn unpack_opt<'c>( arg: ConfiguredTargetListExprArg<'v>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'v>, dice: &mut DiceComputations, allow_unconfigured: bool, @@ -257,13 +258,13 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { match arg { ConfiguredTargetListExprArg::Target(arg) => { Ok( - Self::unpack_literal(arg, target_platform, ctx, dice, allow_unconfigured) + Self::unpack_literal(arg, global_cfg_options, ctx, dice, allow_unconfigured) .await?, ) } ConfiguredTargetListExprArg::List(arg) => { Ok( - Self::unpack_iterable(arg, target_platform, ctx, dice, allow_unconfigured) + Self::unpack_iterable(arg, global_cfg_options, ctx, dice, allow_unconfigured) .await?, ) } @@ -273,29 +274,29 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { pub(crate) async fn unpack<'c>( // TODO(nga): this does not accept unconfigured targets, so should be narrower type here. arg: ConfiguredTargetListExprArg<'v>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'v>, dice: &mut DiceComputations, ) -> anyhow::Result> { - Self::unpack_opt(arg, target_platform, ctx, dice, false).await + Self::unpack_opt(arg, global_cfg_options, ctx, dice, false).await } pub(crate) async fn unpack_allow_unconfigured<'c>( arg: ConfiguredTargetListExprArg<'v>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'v>, dice: &mut DiceComputations, ) -> anyhow::Result> { - Self::unpack_opt(arg, target_platform, ctx, dice, true).await + Self::unpack_opt(arg, global_cfg_options, ctx, dice, true).await } fn check_allow_unconfigured( allow_unconfigured: bool, unconfigured_label: &str, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result<()> { if !allow_unconfigured { - if target_platform.is_none() { + if global_cfg_options.target_platform.is_none() { soft_error!( "bxl_unconfigured_target_in_cquery", TargetExprError::UnconfiguredTargetInCquery(unconfigured_label.to_owned()) @@ -308,7 +309,7 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { async fn unpack_literal( arg: ConfiguredTargetNodeArg<'v>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'_>, dice: &mut DiceComputations, allow_unconfigured: bool, @@ -321,18 +322,18 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { TargetListExpr::One(TargetExpr::Label(Cow::Borrowed(configured_target.label()))), ), ConfiguredTargetNodeArg::Str(s) => { - Self::check_allow_unconfigured(allow_unconfigured, s, target_platform)?; + Self::check_allow_unconfigured(allow_unconfigured, s, global_cfg_options)?; - Self::unpack_string_literal(s, target_platform, ctx, dice, false).await + Self::unpack_string_literal(s, global_cfg_options, ctx, dice, false).await } ConfiguredTargetNodeArg::Unconfigured(label) => { Self::check_allow_unconfigured( allow_unconfigured, &label.label().to_string(), - target_platform, + global_cfg_options, )?; Ok(TargetListExpr::One(TargetExpr::Label(Cow::Owned( - dice.get_configured_target(label.label(), target_platform.as_ref()) + dice.get_configured_target(label.label(), global_cfg_options) .await?, )))) } @@ -343,13 +344,13 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { // but let's support keep_going for string literals for now. pub(crate) async fn unpack_keep_going<'c>( arg: ConfiguredTargetListExprArg<'v>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'v>, dice: &mut DiceComputations, ) -> anyhow::Result> { match arg { ConfiguredTargetListExprArg::Target(ConfiguredTargetNodeArg::Str(val)) => { - Self::unpack_string_literal(val, target_platform, ctx, dice, true).await + Self::unpack_string_literal(val, global_cfg_options, ctx, dice, true).await } _ => Err(TargetExprError::KeepGoingOnlyForStringLiteral.into()), } @@ -358,23 +359,23 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { // Unpack functionality for a string literal, with keep_going support async fn unpack_string_literal( val: &str, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'_>, dice: &mut DiceComputations, keep_going: bool, ) -> anyhow::Result> { match ParsedPattern::::parse_relaxed( - &ctx.target_alias_resolver, + ctx.target_alias_resolver(), // TODO(nga): Parse relaxed relative to cell root is incorrect. - CellPathRef::new(ctx.cell_name, CellRelativePath::empty()), + CellPathRef::new(ctx.cell_name(), CellRelativePath::empty()), val, - &ctx.cell_resolver, + ctx.cell_resolver(), )? { ParsedPattern::Target(pkg, name, TargetPatternExtra) => { let result = match dice .get_configured_target( &TargetLabel::new(pkg, name.as_ref()), - target_platform.as_ref(), + global_cfg_options, ) .await { @@ -405,7 +406,7 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { let maybe_compatible = get_maybe_compatible_targets( dice, loaded_patterns.iter_loaded_targets_by_package(), - target_platform.as_ref(), + global_cfg_options, keep_going, ) .await?; @@ -424,7 +425,7 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { async fn unpack_iterable<'c>( value: ValueOf<'v, ConfiguredTargetListArg<'v>>, - target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, ctx: &BxlContextNoDice<'_>, dice: &mut DiceComputations, allow_unconfigured: bool, @@ -439,10 +440,10 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { Self::check_allow_unconfigured( allow_unconfigured, &node.label().to_string(), - target_platform, + global_cfg_options, )?; anyhow::Ok(TargetExpr::Label(Cow::Owned( - dice.get_configured_target(node.label(), target_platform.as_ref()) + dice.get_configured_target(node.label(), global_cfg_options) .await?, ))) })) @@ -453,9 +454,14 @@ impl<'v> TargetListExpr<'v, ConfiguredTargetNode> { let mut resolved = vec![]; for item in unpack.items { - let unpacked = - Self::unpack_literal(item, target_platform, ctx, dice, allow_unconfigured) - .await?; + let unpacked = Self::unpack_literal( + item, + global_cfg_options, + ctx, + dice, + allow_unconfigured, + ) + .await?; match unpacked { TargetListExpr::One(node) => resolved.push(node), @@ -508,11 +514,11 @@ impl<'v> TargetListExpr<'v, TargetNode> { )), TargetNodeOrTargetLabelOrStr::Str(s) => { match ParsedPattern::::parse_relaxed( - &ctx.target_alias_resolver, + ctx.target_alias_resolver(), // TODO(nga): Parse relaxed relative to cell root is incorrect. - CellPathRef::new(ctx.cell_name, CellRelativePath::empty()), + CellPathRef::new(ctx.cell_name(), CellRelativePath::empty()), s, - &ctx.cell_resolver, + ctx.cell_resolver(), )? { ParsedPattern::Target(pkg, name, TargetPatternExtra) => { Ok(TargetListExpr::One(TargetExpr::Label(Cow::Owned( diff --git a/app/buck2_bxl/src/bxl/starlark_defs/target_universe.rs b/app/buck2_bxl/src/bxl/starlark_defs/target_universe.rs index 3746de136dca8..642b5f35e4d37 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/target_universe.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/target_universe.rs @@ -81,7 +81,7 @@ impl<'v> StarlarkTargetUniverse<'v> { ctx: &'v BxlContext<'v>, target_set: TargetSet, ) -> anyhow::Result> { - let target_universe = CqueryUniverse::build(&target_set).await?; + let target_universe = CqueryUniverse::build(&target_set)?; let target_set = target_universe .get_from_targets(target_set.iter().map(|i| i.label().unconfigured().dupe())); Ok(StarlarkTargetUniverse { diff --git a/app/buck2_bxl/src/bxl/starlark_defs/targetset.rs b/app/buck2_bxl/src/bxl/starlark_defs/targetset.rs index 83dc40ef548cf..dd235d875a3e5 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/targetset.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/targetset.rs @@ -12,7 +12,6 @@ use std::ops::Deref; use allocative::Allocative; use buck2_query::query::environment::QueryTarget; use buck2_query::query::syntax::simple::eval::set::TargetSet; -use buck2_query::query::syntax::simple::eval::set::TargetSetExt; use derive_more::Display; use dupe::Dupe; use starlark::any::ProvidesStaticType; diff --git a/app/buck2_bxl/src/bxl/starlark_defs/uquery.rs b/app/buck2_bxl/src/bxl/starlark_defs/uquery.rs index f6543132709fa..8610bdcfaf3f8 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/uquery.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/uquery.rs @@ -7,15 +7,19 @@ * of this source tree. */ +use std::borrow::Cow; + use allocative::Allocative; use buck2_build_api::query::bxl::BxlUqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_UQUERY_FUNCTIONS; use buck2_build_api::query::oneshot::QUERY_FRONTEND; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_node::nodes::unconfigured::TargetNode; -use buck2_query::query::syntax::simple::eval::set::TargetSetExt; +use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::syntax::simple::functions::helpers::CapturedExpr; use derivative::Derivative; use derive_more::Display; +use dice::DiceComputations; use dupe::Dupe; use futures::FutureExt; use gazebo::prelude::OptionExt; @@ -79,8 +83,8 @@ pub(crate) async fn get_uquery_env<'v>( ) -> anyhow::Result> { (NEW_BXL_UQUERY_FUNCTIONS.get()?)( ctx.project_root().dupe(), - ctx.cell_name, - ctx.cell_resolver.dupe(), + ctx.cell_name(), + ctx.cell_resolver().dupe(), ) .await } @@ -97,6 +101,17 @@ impl<'v> StarlarkUQueryCtx<'v> { } } +async fn unpack_targets<'c, 'v>( + this: &'c StarlarkUQueryCtx<'v>, + dice: &'c mut DiceComputations, + targets: TargetListExprArg<'v>, +) -> anyhow::Result>> { + TargetListExpr::<'v, TargetNode>::unpack(targets, &this.ctx.data, dice) + .await? + .get(dice) + .await +} + /// The context for performing `uquery` operations in bxl. The functions offered on this ctx are /// the same behaviour as the query functions available within uquery command. #[starlark_module] @@ -110,14 +125,8 @@ fn uquery_methods(builder: &mut MethodsBuilder) { this.ctx.via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let from = TargetListExpr::<'v, TargetNode>::unpack(from, ctx, dice) - .await? - .get(dice) - .await?; - let to = TargetListExpr::<'v, TargetNode>::unpack(to, ctx, dice) - .await? - .get(dice) - .await?; + let from = unpack_targets(this, dice, from).await?; + let to = unpack_targets(this, dice, to).await?; get_uquery_env(ctx) .await? .allpaths(dice, &from, &to) @@ -138,14 +147,8 @@ fn uquery_methods(builder: &mut MethodsBuilder) { this.ctx.via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let from = TargetListExpr::<'v, TargetNode>::unpack(from, ctx, dice) - .await? - .get(dice) - .await?; - let to = TargetListExpr::<'v, TargetNode>::unpack(to, ctx, dice) - .await? - .get(dice) - .await?; + let from = unpack_targets(this, dice, from).await?; + let to = unpack_targets(this, dice, to).await?; get_uquery_env(ctx) .await? .somepath(dice, &from, &to) @@ -164,13 +167,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { value: &str, targets: TargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; targets .attrfilter(attr, &|v| Ok(v == value)) .map(StarlarkTargetSet::from) @@ -193,13 +193,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { targets: TargetListExprArg<'v>, ) -> anyhow::Result { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; targets.inputs() } .boxed_local() @@ -221,13 +218,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { regex: &str, targets: TargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; targets.kind(regex).map(StarlarkTargetSet::from) } .boxed_local() @@ -257,10 +251,7 @@ fn uquery_methods(builder: &mut MethodsBuilder) { .into_option() .try_map(buck2_query_parser::parse_expr)?; - let targets = TargetListExpr::<'v, TargetNode>::unpack(universe, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, universe).await?; get_uquery_env(ctx) .await? @@ -299,16 +290,8 @@ fn uquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let universe = - TargetListExpr::<'v, TargetNode>::unpack(universe, ctx, dice) - .await? - .get(dice) - .await?; - - let targets = TargetListExpr::<'v, TargetNode>::unpack(from, ctx, dice) - .await? - .get(dice) - .await?; + let universe = unpack_targets(this, dice, universe).await?; + let targets = unpack_targets(this, dice, from).await?; get_uquery_env(ctx) .await? @@ -335,13 +318,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { targets: TargetListExprArg<'v>, ) -> anyhow::Result> { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; targets.filter_name(regex) } .boxed_local() @@ -366,10 +346,7 @@ fn uquery_methods(builder: &mut MethodsBuilder) { .via_dice(|mut dice, ctx| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; get_uquery_env(ctx).await?.testsof(dice, &targets).await } .boxed_local() @@ -392,15 +369,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { targets: TargetListExprArg<'v>, ) -> anyhow::Result { this.ctx - .via_dice(|mut dice, ctx| { + .via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = - &*TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; - + let targets = unpack_targets(this, dice, targets).await?; Ok(targets.buildfile()) } .boxed_local() @@ -430,7 +402,7 @@ fn uquery_methods(builder: &mut MethodsBuilder) { async { get_uquery_env(ctx) .await? - .owner(dice, (files.get(ctx).await?).as_ref()) + .owner(dice, (files.get(&this.ctx.data).await?).as_ref()) .await } .boxed_local() @@ -453,13 +425,10 @@ fn uquery_methods(builder: &mut MethodsBuilder) { value: &str, targets: TargetListExprArg<'v>, ) -> anyhow::Result> { - this.ctx.via_dice(|mut dice, ctx| { + this.ctx.via_dice(|mut dice, _| { dice.via(|dice| { async { - let targets = TargetListExpr::<'v, TargetNode>::unpack(targets, ctx, dice) - .await? - .get(dice) - .await?; + let targets = unpack_targets(this, dice, targets).await?; targets .attrregexfilter(attribute, value) .map(StarlarkTargetSet::from) @@ -499,7 +468,13 @@ fn uquery_methods(builder: &mut MethodsBuilder) { parse_query_evaluation_result( QUERY_FRONTEND .get()? - .eval_uquery(dice, &this.ctx.working_dir()?, query, &query_args, None) + .eval_uquery( + dice, + &this.ctx.working_dir()?, + query, + &query_args, + GlobalCfgOptions::default(), + ) .await?, eval, ) diff --git a/app/buck2_bxl/src/bxl/value_as_starlark_target_label.rs b/app/buck2_bxl/src/bxl/value_as_starlark_target_label.rs index e9601fb6a2b3f..bba5c6fecf1b8 100644 --- a/app/buck2_bxl/src/bxl/value_as_starlark_target_label.rs +++ b/app/buck2_bxl/src/bxl/value_as_starlark_target_label.rs @@ -29,7 +29,7 @@ pub(crate) enum ValueAsStarlarkTargetLabel<'v> { } impl<'v> ValueAsStarlarkTargetLabel<'v> { - pub const NONE: Self = Self::None(NoneType); + pub(crate) const NONE: Self = Self::None(NoneType); pub(crate) fn parse_target_platforms( self, diff --git a/app/buck2_bxl/src/command.rs b/app/buck2_bxl/src/command.rs index 3d3875957e28a..1300b892b91f8 100644 --- a/app/buck2_bxl/src/command.rs +++ b/app/buck2_bxl/src/command.rs @@ -49,7 +49,7 @@ use buck2_interpreter::paths::bxl::BxlFilePath; use buck2_interpreter::paths::module::StarlarkModulePath; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceComputations; @@ -133,8 +133,8 @@ async fn bxl( let project_root = server_ctx.project_root().to_string(); let client_ctx = request.client_context()?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let bxl_args = match get_bxl_cli_args(cwd, &ctx, &bxl_label, &request.bxl_args, &cell_resolver).await? { @@ -156,8 +156,8 @@ async fn bxl( let bxl_key = BxlKey::new( bxl_label.clone(), bxl_args, - global_target_platform, request.print_stacktrace, + global_cfg_options, ); let ctx = &ctx; @@ -183,8 +183,15 @@ async fn bxl( // in a separate instance of the map. &Arc::new((*materializations).clone()), ); - - let build_result = ensure_artifacts(ctx, &materialization_context, &bxl_result).await; + let build_results = bxl_result.get_build_result_opt(); + let configured_build_results = filter_bxl_build_results(build_results); + let build_result = ensure_artifacts( + ctx, + &materialization_context, + &configured_build_results, + bxl_result.get_artifacts_opt(), + ) + .await; copy_output(stdout, ctx, bxl_result.get_output_loc()).await?; copy_output(server_ctx.stderr()?, ctx, bxl_result.get_error_loc()).await?; @@ -258,36 +265,36 @@ async fn copy_output( async fn ensure_artifacts( ctx: &DiceComputations, materialization_ctx: &MaterializationContext, - bxl_result: &buck2_build_api::bxl::result::BxlResult, + target_results: &[&ConfiguredBuildTargetResult], + artifacts: Option<&Vec>, ) -> Result<(), Vec> { - match bxl_result { - buck2_build_api::bxl::result::BxlResult::None { .. } => Ok(()), - buck2_build_api::bxl::result::BxlResult::BuildsArtifacts { - built, artifacts, .. - } => { + if let Some(artifacts) = artifacts { + return { get_dispatcher() .span_async(BxlEnsureArtifactsStart {}, async move { ( - ensure_artifacts_inner(ctx, materialization_ctx, built, artifacts).await, + ensure_artifacts_inner(ctx, materialization_ctx, target_results, artifacts) + .await, BxlEnsureArtifactsEnd {}, ) }) .await - } + }; } + Ok(()) } async fn ensure_artifacts_inner( ctx: &DiceComputations, materialization_ctx: &MaterializationContext, - built: &[BxlBuildResult], + target_results: &[&ConfiguredBuildTargetResult], artifacts: &[ArtifactGroup], ) -> Result<(), Vec> { let mut futs = vec![]; - built.iter().for_each(|res| match res { - BxlBuildResult::Built(ConfiguredBuildTargetResult { outputs, .. }) => { - outputs.iter().for_each(|res| match res { + target_results.iter().for_each(|res| { + for res in &res.outputs { + match res { Ok(artifacts) => { for (artifact, _value) in artifacts.values.iter() { futs.push( @@ -305,10 +312,8 @@ async fn ensure_artifacts_inner( } } Err(e) => futs.push(futures::future::ready(Err(e.dupe())).boxed()), - }); + } } - - BxlBuildResult::None => {} }); artifacts.iter().for_each(|a| { @@ -386,3 +391,18 @@ pub(crate) fn parse_bxl_label_from_cli( name: bxl_fn.to_owned(), }) } + +fn filter_bxl_build_results( + build_results: Option<&Vec>, +) -> Vec<&ConfiguredBuildTargetResult> { + if let Some(build_results) = build_results { + return build_results + .iter() + .filter_map(|res| match res { + BxlBuildResult::Built { result, .. } => Some(result), + BxlBuildResult::None => None, + }) + .collect(); + } + vec![] +} diff --git a/app/buck2_bxl/src/profile_command.rs b/app/buck2_bxl/src/profile_command.rs index 75cbe9cc3a128..7511484235091 100644 --- a/app/buck2_bxl/src/profile_command.rs +++ b/app/buck2_bxl/src/profile_command.rs @@ -24,7 +24,7 @@ use buck2_profile::starlark_profiler_configuration_from_request; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; @@ -103,15 +103,16 @@ impl ServerCommandTemplate for BxlProfileServerCommand { }; let client_ctx = self.req.client_context()?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx) - .await?; + let global_cfg_options = global_cfg_options_from_client_context( + client_ctx, server_ctx, &mut ctx, + ) + .await?; let bxl_key = BxlKey::new( bxl_label.clone(), bxl_args, - global_target_platform, /* force print stacktrace */ false, + global_cfg_options, ); server_ctx diff --git a/app/buck2_cfg_constructor/src/calculation.rs b/app/buck2_cfg_constructor/src/calculation.rs index ef351201eed50..63923cae3cb96 100644 --- a/app/buck2_cfg_constructor/src/calculation.rs +++ b/app/buck2_cfg_constructor/src/calculation.rs @@ -23,7 +23,8 @@ use buck2_node::cfg_constructor::CfgConstructorCalculationImpl; use buck2_node::cfg_constructor::CfgConstructorImpl; use buck2_node::cfg_constructor::CFG_CONSTRUCTOR_CALCULATION_IMPL; use buck2_node::metadata::value::MetadataValue; -use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeRef; +use buck2_node::rule_type::RuleType; use buck2_node::super_package::SuperPackage; use derive_more::Display; use dice::CancellationContext; @@ -81,9 +82,11 @@ impl CfgConstructorCalculationImpl for CfgConstructorCalculationInstance { async fn eval_cfg_constructor( &self, ctx: &DiceComputations, - target: &TargetNode, + target: TargetNodeRef<'_>, super_package: &SuperPackage, cfg: ConfigurationData, + cli_modifiers: &Arc>, + rule_type: &RuleType, ) -> anyhow::Result { #[derive(Clone, Display, Dupe, Debug, Eq, Hash, PartialEq, Allocative)] #[display(fmt = "CfgConstructorInvocationKey")] @@ -91,6 +94,8 @@ impl CfgConstructorCalculationImpl for CfgConstructorCalculationInstance { package_cfg_modifiers: Option, target_cfg_modifiers: Option, cfg: ConfigurationData, + cli_modifiers: Arc>, + rule_type: RuleType, } #[async_trait] @@ -114,7 +119,8 @@ impl CfgConstructorCalculationImpl for CfgConstructorCalculationInstance { &self.cfg, self.package_cfg_modifiers.as_ref(), self.target_cfg_modifiers.as_ref(), - &[], + &self.cli_modifiers, + &self.rule_type, ) .await .map_err(buck2_error::Error::from) @@ -139,9 +145,12 @@ impl CfgConstructorCalculationImpl for CfgConstructorCalculationInstance { .get_package_value_json(modifier_key)? .map(MetadataValue::new); let target_cfg_modifiers = target.metadata()?.and_then(|m| m.get(modifier_key)).duped(); - // If there are no PACKAGE or target modifiers, return the original configuration without computing DICE call + // If there are no PACKAGE/target/cli modifiers, return the original configuration without computing DICE call // TODO(scottcao): This is just for rollout purpose. Remove once modifier is rolled out - if package_cfg_modifiers.is_none() && target_cfg_modifiers.is_none() { + if package_cfg_modifiers.is_none() + && target_cfg_modifiers.is_none() + && cli_modifiers.is_empty() + { return Ok(cfg); } @@ -149,6 +158,8 @@ impl CfgConstructorCalculationImpl for CfgConstructorCalculationInstance { package_cfg_modifiers, target_cfg_modifiers, cfg, + cli_modifiers: cli_modifiers.dupe(), + rule_type: rule_type.dupe(), }; Ok(ctx.compute(&key).await??) } diff --git a/app/buck2_cfg_constructor/src/lib.rs b/app/buck2_cfg_constructor/src/lib.rs index c364dbea6c0dc..a10d22a62c582 100644 --- a/app/buck2_cfg_constructor/src/lib.rs +++ b/app/buck2_cfg_constructor/src/lib.rs @@ -9,7 +9,6 @@ #![feature(error_generic_member_access)] #![feature(async_closure)] -#![feature(async_fn_in_trait)] pub(crate) mod calculation; pub(crate) mod registration; @@ -39,6 +38,7 @@ use buck2_node::metadata::key::MetadataKeyRef; use buck2_node::metadata::value::MetadataValue; use buck2_node::nodes::frontend::TargetGraphCalculation; use buck2_node::nodes::unconfigured::RuleKind; +use buck2_node::rule_type::RuleType; use calculation::CfgConstructorCalculationInstance; use dice::DiceComputations; use futures::future::try_join_all; @@ -75,6 +75,7 @@ async fn eval_pre_constraint_analysis<'v>( package_cfg_modifiers: Option<&MetadataValue>, target_cfg_modifiers: Option<&MetadataValue>, cli_modifiers: &[String], + rule_type: &RuleType, module: &'v Module, print: &'v EventDispatcherPrintHandler, ) -> anyhow::Result<(Vec, Value<'v>, Evaluator<'v, 'v>)> { @@ -99,6 +100,7 @@ async fn eval_pre_constraint_analysis<'v>( .alloc(package_cfg_modifiers.map(|m| m.as_json())); let target_cfg_modifiers = eval.heap().alloc(target_cfg_modifiers.map(|m| m.as_json())); let cli_modifiers = eval.heap().alloc(cli_modifiers); + let rule_name = eval.heap().alloc(rule_type.name()); // TODO: should eventually accept cli modifiers and target modifiers (T163570597) let pre_constraint_analysis_args = vec![ @@ -106,6 +108,7 @@ async fn eval_pre_constraint_analysis<'v>( ("package_modifiers", package_cfg_modifiers), ("target_modifiers", target_cfg_modifiers), ("cli_modifiers", cli_modifiers), + ("rule_name", rule_name), ]; // Type check + unpack @@ -211,6 +214,7 @@ async fn eval_underlying( package_cfg_modifiers: Option<&MetadataValue>, target_cfg_modifiers: Option<&MetadataValue>, cli_modifiers: &[String], + rule_type: &RuleType, ) -> anyhow::Result { let module = Module::new(); let print = EventDispatcherPrintHandler(get_dispatcher()); @@ -225,6 +229,7 @@ async fn eval_underlying( package_cfg_modifiers, target_cfg_modifiers, cli_modifiers, + rule_type, &module, &print, ) @@ -255,6 +260,7 @@ impl CfgConstructorImpl for CfgConstructor { package_cfg_modifiers: Option<&'a MetadataValue>, target_cfg_modifiers: Option<&'a MetadataValue>, cli_modifiers: &'a [String], + rule_type: &'a RuleType, ) -> Pin> + Send + 'a>> { // Get around issue of Evaluator not being send by wrapping future in UnsafeSendFuture let fut = async move { @@ -265,6 +271,7 @@ impl CfgConstructorImpl for CfgConstructor { package_cfg_modifiers, target_cfg_modifiers, cli_modifiers, + rule_type, ) .await }; diff --git a/app/buck2_cli_proto/daemon.proto b/app/buck2_cli_proto/daemon.proto index 4f459d1791814..66bbde64aabfc 100644 --- a/app/buck2_cli_proto/daemon.proto +++ b/app/buck2_cli_proto/daemon.proto @@ -75,6 +75,7 @@ message StatusResponse { string isolation_dir = 10; optional uint32 forkserver_pid = 11; optional bool supports_vpnless = 12; + optional bool http2 = 13; } message PingRequest { @@ -126,6 +127,7 @@ message ClientContext { /// Contents of `BUCK2_HARD_ERROR` environment variable. string buck2_hard_error = 20; + repeated string cli_modifiers = 21; } message TargetsRequest { @@ -343,6 +345,7 @@ message CommonBuildOptions { // that should be handled in the server or the client, though. bool unstable_print_build_report = 4242000; string unstable_build_report_filename = 4242003; + bool unstable_include_failures_build_report = 4242004; } message BuildRequest { @@ -426,6 +429,12 @@ message TestRequest { CommonBuildOptions build_opts = 9; TestSessionOptions session_options = 11; + + // How long to execute tests for? If the timeout is exceeded, Buck2 will exit + // as quickly as possible and not run further tests. In-flight tests will be + // cancelled. The test orchestrator will be allowed to shut down gracefully. + // The exit code will be a user failure. + optional google.protobuf.Duration timeout = 12; } message BxlRequest { @@ -792,7 +801,7 @@ message FileStatusRequest { // The paths we want to learn about repeated string paths = 2; // Show hashes of files passed in. - bool verbose = 3; + bool show_matches = 3; } message FlushDepFilesRequest {} @@ -880,7 +889,7 @@ service DaemonApi { // Crashes the Buck daemon. Unless you are writing tests or checking Buck2's // panic behavior, you probably don't want this. - rpc Unstable_Crash(UnstableCrashRequest) returns (UnstableCrashResponse); + rpc Unstable_Crash(UnstableCrashRequest) returns (CommandResult); // Crashes the Buck daemon with a segfault. Unless you are writing tests or // checking Buck2's segfault behavior, you probably don't want this. diff --git a/app/buck2_cli_proto/src/protobuf_util.rs b/app/buck2_cli_proto/src/protobuf_util.rs index 61d760053da34..4a5740062f5cd 100644 --- a/app/buck2_cli_proto/src/protobuf_util.rs +++ b/app/buck2_cli_proto/src/protobuf_util.rs @@ -45,7 +45,7 @@ impl Decoder for ProtobufSplitter { } #[cfg(test)] -mod test { +mod tests { use anyhow::Context as _; use futures::stream::StreamExt; use prost::Message; diff --git a/app/buck2_client/src/commands/build/mod.rs b/app/buck2_client/src/commands/build/mod.rs index 130f32abfed8d..a3b920ca5538b 100644 --- a/app/buck2_client/src/commands/build/mod.rs +++ b/app/buck2_client/src/commands/build/mod.rs @@ -371,7 +371,7 @@ pub(crate) fn print_outputs( } #[cfg(test)] -mod test { +mod tests { use assert_matches::assert_matches; use build_providers::Action; use clap::Parser; diff --git a/app/buck2_client/src/commands/build/out.rs b/app/buck2_client/src/commands/build/out.rs index 1f8db6196726a..577c392531197 100644 --- a/app/buck2_client/src/commands/build/out.rs +++ b/app/buck2_client/src/commands/build/out.rs @@ -201,7 +201,7 @@ fn convert_broken_pipe_error(e: io::Error) -> anyhow::Error { } #[cfg(test)] -mod test { +mod tests { #[cfg(unix)] mod unix { use std::path::Path; diff --git a/app/buck2_client/src/commands/debug/chrome_trace.rs b/app/buck2_client/src/commands/debug/chrome_trace.rs index 7460ac6efb568..6dbbcfc3c4e89 100644 --- a/app/buck2_client/src/commands/debug/chrome_trace.rs +++ b/app/buck2_client/src/commands/debug/chrome_trace.rs @@ -366,7 +366,7 @@ where for (key, counter) in self.counters.iter_mut() { // TODO: With float counters this equality comparison seems sketchy. if counter.value == self.zero_value { - // If the counter is currently at its zero value, then emit the zero once, and thne + // If the counter is currently at its zero value, then emit the zero once, and then // stop emitting this counter altogether. if !counter.implicitly_zero { counters_to_output[key] = json!(counter.value); @@ -651,7 +651,7 @@ impl ChromeTraceWriter { }); enum Categorization<'a> { - /// Show this node on a speciifc tack + /// Show this node on a specific track Show { category: SpanCategorization, name: Cow<'a, str>, diff --git a/app/buck2_client/src/commands/debug/crash.rs b/app/buck2_client/src/commands/debug/crash.rs index 1b7d52fd4b24d..c75719b9bafe5 100644 --- a/app/buck2_client/src/commands/debug/crash.rs +++ b/app/buck2_client/src/commands/debug/crash.rs @@ -18,7 +18,11 @@ use buck2_client_ctx::exit_result::ExitResult; use buck2_client_ctx::streaming::StreamingCommand; #[derive(Debug, clap::Parser)] -pub struct CrashCommand {} +pub struct CrashCommand { + /// Event-log options. + #[clap(flatten)] + pub event_log_opts: CommonDaemonCommandOptions, +} #[async_trait] impl StreamingCommand for CrashCommand { @@ -46,7 +50,7 @@ impl StreamingCommand for CrashCommand { } fn event_log_opts(&self) -> &CommonDaemonCommandOptions { - CommonDaemonCommandOptions::default_ref() + &self.event_log_opts } fn common_opts(&self) -> &CommonBuildConfigurationOptions { diff --git a/app/buck2_client/src/commands/debug/file_status.rs b/app/buck2_client/src/commands/debug/file_status.rs index 76afa28ecd893..5b1fb124e0ab4 100644 --- a/app/buck2_client/src/commands/debug/file_status.rs +++ b/app/buck2_client/src/commands/debug/file_status.rs @@ -30,8 +30,8 @@ pub struct FileStatusCommand { #[clap(value_name = "PATH", required = true)] paths: Vec, - #[clap(long, short)] - verbose: bool, + #[clap(long, short, help = "Print all matches")] + show_matches: bool, } #[async_trait] @@ -57,7 +57,7 @@ impl StreamingCommand for FileStatusCommand { paths: self .paths .try_map(|x| x.resolve(&ctx.working_dir).into_string())?, - verbose: self.verbose, + show_matches: self.show_matches, }, ctx.stdin() .console_interaction_stream(&self.common_opts.console_opts), diff --git a/app/buck2_client/src/commands/debug/persist_event_logs.rs b/app/buck2_client/src/commands/debug/persist_event_logs.rs index a85c1869a516c..6d9e3cda3e8db 100644 --- a/app/buck2_client/src/commands/debug/persist_event_logs.rs +++ b/app/buck2_client/src/commands/debug/persist_event_logs.rs @@ -21,7 +21,7 @@ use buck2_core::fs::paths::abs_path::AbsPathBuf; use buck2_core::soft_error; use buck2_data::instant_event::Data; use buck2_data::InstantEvent; -use buck2_data::PersistSubprocess; +use buck2_data::PersistEventLogSubprocess; use buck2_events::sink::scribe::new_thrift_scribe_sink_if_enabled; use buck2_events::sink::scribe::ThriftScribeSink; use buck2_events::BuckEvent; @@ -72,25 +72,38 @@ impl PersistEventLogsCommand { let sink = create_scribe_sink(&ctx)?; ctx.with_runtime(async move |mut ctx| { let mut stdin = io::BufReader::new(ctx.stdin()); - if let Err(e) = self.write_and_upload(&mut stdin).await { - dispatch_event_to_scribe( - sink.as_ref(), - &ctx.trace_id, - PersistSubprocess { - errors: vec![e.to_string()], - }, - ) - .await; - let _res = soft_error!(categorize_error(&e), e); + let allow_vpnless = self.allow_vpnless; + let (local_result, remote_result) = self.write_and_upload(&mut stdin).await; + + let (local_error_messages, local_error_category, local_success) = + status_from_result(local_result); + let (remote_error_messages, remote_error_category, remote_success) = + status_from_result(remote_result); + + let event_to_send = PersistEventLogSubprocess { + local_error_messages, + local_error_category, + local_success, + remote_error_messages, + remote_error_category, + remote_success, + allow_vpnless, + metadata: buck2_events::metadata::collect(), }; + dispatch_event_to_scribe(sink.as_ref(), &ctx.trace_id, event_to_send).await; }); ExitResult::success() } - async fn write_and_upload(self, stdin: impl io::AsyncBufRead + Unpin) -> anyhow::Result<()> { + async fn write_and_upload( + self, + stdin: impl io::AsyncBufRead + Unpin, + ) -> (anyhow::Result<()>, anyhow::Result<()>) { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let file = Mutex::new(create_log_file(self.local_path).await?); - + let file = match create_log_file(self.local_path).await { + Ok(f) => Mutex::new(f), + Err(e) => return (Err(e), Err(anyhow::anyhow!("Not tried"))), + }; let write = write_task(&file, tx, stdin); let upload = upload_task( &file, @@ -102,9 +115,7 @@ impl PersistEventLogsCommand { // Wait for both tasks to finish. If the upload fails we want to keep writing to disk let (write_result, upload_result) = tokio::join!(write, upload); - write_result?; - upload_result?; - Ok(()) + (write_result, upload_result) } } @@ -278,6 +289,21 @@ async fn write_to_file( Ok(()) } +fn status_from_result(res: anyhow::Result<()>) -> (Vec, Option, bool) { + // Returns a tuple of error messages, error category, and success/failure + if let Err(e) = res { + let status = ( + vec![e.to_string()], + Some(categorize_error(&e).to_owned()), + false, + ); + let _unused = soft_error!(categorize_error(&e), e); + status + } else { + (vec![], None, true) + } +} + fn categorize_error(err: &anyhow::Error) -> &'static str { // This is for internal error tracking in `logview buck2` // Each category should point to 1 root cause @@ -310,9 +336,9 @@ fn categorize_error(err: &anyhow::Error) -> &'static str { async fn dispatch_event_to_scribe( sink: Option<&ThriftScribeSink>, invocation_id: &TraceId, - result: PersistSubprocess, + result: PersistEventLogSubprocess, ) { - let data = Some(Data::PersistSubprocess(result)); + let data = Some(Data::PersistEventLogSubprocess(result)); let event = InstantEvent { data }; if let Some(sink) = sink { sink.send_now(BuckEvent::new( diff --git a/app/buck2_client/src/commands/log/replay.rs b/app/buck2_client/src/commands/log/replay.rs index ca2f7fa348219..ff989b55a6f91 100644 --- a/app/buck2_client/src/commands/log/replay.rs +++ b/app/buck2_client/src/commands/log/replay.rs @@ -16,6 +16,7 @@ use buck2_client_ctx::exit_result::ExitResult; use buck2_client_ctx::replayer::Replayer; use buck2_client_ctx::signal_handler::with_simple_sigint_handler; use buck2_client_ctx::subscribers::get::get_console_with_root; +use buck2_client_ctx::subscribers::subscribers::EventSubscribers; use crate::commands::log::options::EventLogOptions; @@ -73,7 +74,7 @@ impl ReplayCommand { console_opts.superconsole_config(), )?; - let res = EventsCtx::new(vec![console]) + let res = EventsCtx::new(EventSubscribers::new(vec![console])) .unpack_stream::<_, ReplayResult, _>( &mut NoPartialResultHandler, Box::pin(replayer), diff --git a/app/buck2_client/src/commands/log/summary.rs b/app/buck2_client/src/commands/log/summary.rs index b12b097f99f1d..9a8bd267f5422 100644 --- a/app/buck2_client/src/commands/log/summary.rs +++ b/app/buck2_client/src/commands/log/summary.rs @@ -98,6 +98,7 @@ impl SummaryCommand { "Showing summary from: {}", invocation.display_command_line() )?; + buck2_client_ctx::eprintln!("build ID: {}", invocation.trace_id)?; let mut stats = Stats::default(); diff --git a/app/buck2_client/src/commands/profile.rs b/app/buck2_client/src/commands/profile.rs index 1f16cf7cdaed3..c4c59cfb5c740 100644 --- a/app/buck2_client/src/commands/profile.rs +++ b/app/buck2_client/src/commands/profile.rs @@ -155,7 +155,7 @@ pub struct ProfileCommonOptions { /// This is probably what you want when profiling analysis. /// /// `-allocated` means allocated memory, including memory which is later garbage collected. - #[clap(long, short = 'm', value_enum)] + #[clap(long, value_enum)] mode: BuckProfileMode, } diff --git a/app/buck2_client/src/commands/query/aquery.rs b/app/buck2_client/src/commands/query/aquery.rs index 24df041bc4492..67712f867f16e 100644 --- a/app/buck2_client/src/commands/query/aquery.rs +++ b/app/buck2_client/src/commands/query/aquery.rs @@ -19,32 +19,53 @@ use buck2_client_ctx::daemon::client::BuckdClientConnector; use buck2_client_ctx::daemon::client::StdoutPartialResultHandler; use buck2_client_ctx::exit_result::ExitResult; use buck2_client_ctx::streaming::StreamingCommand; +use buck2_core::if_else_opensource; use crate::commands::query::common::CommonQueryOptions; -/// Perform queries on the action graph (experimental). -/// -/// The action graph consists of all the declared actions for a build, with dependencies -/// when one action consumes the outputs of another action. -/// -/// Run `buck2 docs aquery` for more documentation about the functions available in aquery -/// -/// Examples: -/// -/// Print the action producing a target's default output -/// -/// `buck2 aquery //java/com/example/app:amazing` -/// -/// List all the commands for run actions for building a target -/// -/// `buck2 aquery 'kind(run, deps("//java/com/example/app:amazing+more"))' --output-attribute=cmd` -/// -/// Dynamic outputs (`ctx.actions.dynamic_output`): -/// -/// Currently, aquery interacts poorly with dynamic outputs. It may return incorrect results or otherwise -/// behave unexpectedly. +fn help() -> &'static str { + concat!( + r#"Perform queries on the action graph (experimental) + +The action graph consists of all the declared actions for a build, +with dependencies when one action consumes the outputs of another +action. + +Run `buck2 docs aquery` or +"#, + if_else_opensource!( + "https://buck2.build/docs/users/query/aquery/", + "https://www.internalfb.com/intern/staticdocs/buck2/docs/users/query/aquery/", + ), + r#" +for more documentation about the functions available in aquery +expressions. + +Examples: + +Print the action producing a target's default output + +`buck2 aquery //java/com/example/app:amazing` + +List all the commands for run actions for building a target + +`buck2 aquery 'kind(run, deps("//java/com/example/app:amazing+more"))' --output-attribute=cmd` + +Dynamic outputs (`ctx.actions.dynamic_output`): + +Currently, aquery interacts poorly with dynamic outputs. It may +return incorrect results or otherwise behave unexpectedly. +"# + ) +} + #[derive(Debug, clap::Parser)] -#[clap(name = "aquery")] +#[clap( + name = "aquery", + about = "Perform queries on the action graph (experimental)", + long_about = help(), + verbatim_doc_comment, +)] pub struct AqueryCommand { #[clap(flatten)] common_opts: CommonCommandOptions, diff --git a/app/buck2_client/src/commands/query/cquery.rs b/app/buck2_client/src/commands/query/cquery.rs index b98b477b1e46f..0a1aa4da5bb1f 100644 --- a/app/buck2_client/src/commands/query/cquery.rs +++ b/app/buck2_client/src/commands/query/cquery.rs @@ -19,36 +19,58 @@ use buck2_client_ctx::daemon::client::BuckdClientConnector; use buck2_client_ctx::daemon::client::StdoutPartialResultHandler; use buck2_client_ctx::exit_result::ExitResult; use buck2_client_ctx::streaming::StreamingCommand; +use buck2_core::if_else_opensource; use crate::commands::query::common::CommonQueryOptions; -/// Perform queries on the configured target graph. -/// -/// The configured target graph includes information about the configuration (platforms) and -/// transitions involved in building targets. In the configured graph, `selects` are fully -/// resolved. The same target may appear in multiple different configurations (when printed, -/// the configuration is after the target in parentheses). -/// -/// A user can specify a `--target-universe` flag to control how literals are resolved. When -/// provided, any literals will resolve to all matching targets within the universe (which -/// includes the targets passed as the universe and all transitive deps of them). -/// When not provided, we implicitly set the universe to be rooted at every target literal -/// in the `cquery`. -/// -/// Run `buck2 docs cquery` for more documentation about the functions available in cquery -/// expressions. -/// -/// Examples: -/// -/// Print all the attributes of a target -/// -/// `buck2 cquery //java/com/example/app:amazing --output-all-attributes` -/// -/// List the deps of a target (special characters in a target will require quotes): -/// -/// `buck2 cquery 'deps("//java/com/example/app:amazing+more")'` +fn help() -> &'static str { + concat!( + r#"Perform queries on the configured target graph + +The configured target graph includes information about the configuration +(platforms) and transitions involved in building targets. In the +configured graph, `selects` are fully resolved. The same target may +appear in multiple different configurations (when printed, the +configuration is after the target in parentheses). + +A user can specify a `--target-universe` flag to control how literals +are resolved. When provided, any literals will resolve to all +matching targets within the universe (which includes the targets +passed as the universe and all transitive deps of them). When not +provided, we implicitly set the universe to be rooted at every +target literal in the `cquery`. + +Run `buck2 docs cquery` or +"#, + if_else_opensource!( + "https://buck2.build/docs/users/query/cquery/", + "https://www.internalfb.com/intern/staticdocs/buck2/docs/users/query/cquery/", + ), + r#" +for more documentation about the functions available in cquery +expressions. + +Examples: + +Print all the attributes of a target + +`buck2 cquery //java/com/example/app:amazing --output-all-attributes` + +List the deps of a target (special characters in a target will +require quotes): + +`buck2 cquery 'deps("//java/com/example/app:amazing+more")'` +"# + ) +} + #[derive(Debug, clap::Parser)] -#[clap(name = "cquery")] +#[clap( + name = "cquery", + about = "Perform queries on the configured target graph", + long_about = help(), + verbatim_doc_comment, +)] pub struct CqueryCommand { #[clap(flatten)] common_opts: CommonCommandOptions, diff --git a/app/buck2_client/src/commands/query/uquery.rs b/app/buck2_client/src/commands/query/uquery.rs index 903f987c5ebd0..78e3594ed36c0 100644 --- a/app/buck2_client/src/commands/query/uquery.rs +++ b/app/buck2_client/src/commands/query/uquery.rs @@ -19,40 +19,61 @@ use buck2_client_ctx::daemon::client::BuckdClientConnector; use buck2_client_ctx::daemon::client::StdoutPartialResultHandler; use buck2_client_ctx::exit_result::ExitResult; use buck2_client_ctx::streaming::StreamingCommand; +use buck2_core::if_else_opensource; use crate::commands::query::common::CommonQueryOptions; -/// Perform queries on the unconfigured target graph. -/// -/// The unconfigured target graph consists of the targets as they are defined in the build -/// files. In this graph, each target appears exactly once and `select()`s are in the unresolved -/// form. For large queries, the unconfigured graph may be much smaller than the configured -/// graph and queries can be much more efficiently performed there. -/// -/// When querying the unconfigured graph, dependencies appearing in all branches of `select()` -/// dictionaries will be treated as dependencies. -/// -/// Run `buck2 docs uquery` for more documentation about the functions available in cquery -/// expressions. -/// -/// Examples: -/// -/// Print all the attributes of a target -/// -/// `buck2 uquery //java/com/example/app:amazing --output-all-attributes -/// -/// List the deps of a target (special characters in a target will require quotes): -/// `buck2 uquery 'deps("//java/com/example/app:amazing+more")'` -/// -/// select() encoding: -/// -/// When printed, values with `select()`s use a special json encoding. -/// -/// `1 + select({"//:a": 1, "DEFAULT": 2})` will be encoded as: -/// -/// `{"__type": "concat", "items": [1, {"__type": "selector", "entries": {"//:a": 1, "DEFAULT": 2}}]}` +fn help() -> &'static str { + concat!( + "Perform queries on the unconfigured target graph + +The unconfigured target graph consists of the targets as they are +defined in the build files. In this graph, each target appears +exactly once and `select()`s are in the unresolved form. For large +queries, the unconfigured graph may be much smaller than the +configured graph and queries can be much more efficiently performed +there. + +When querying the unconfigured graph, dependencies appearing in all +branches of `select()` dictionaries will be treated as dependencies. + +Run `buck2 docs uquery` or +", + if_else_opensource!( + "https://buck2.build/docs/users/query/uquery/", + "https://www.internalfb.com/intern/staticdocs/buck2/docs/users/query/uquery/", + ), + r#" +for more documentation about the functions available in uquery +expressions. + +Examples: + +Print all the attributes of a target + +`buck2 uquery //java/com/example/app:amazing --output-all-attributes + +List the deps of a target (special characters in a target will require quotes): +`buck2 uquery 'deps("//java/com/example/app:amazing+more")'` + +select() encoding: + +When printed, values with `select()`s use a special json encoding. + +`1 + select({"//:a": 1, "DEFAULT": 2})` will be encoded as: + +`{"__type": "concat", "items": [1, {"__type": "selector", "entries": {"//:a": 1, "DEFAULT": 2}}]}` +"# + ) +} + #[derive(Debug, clap::Parser)] -#[clap(name = "uquery")] +#[clap( + name = "uquery", + about = "Perform queries on the unconfigured target graph", + long_about = help(), + verbatim_doc_comment, +)] pub struct UqueryCommand { #[clap(flatten)] common_opts: CommonCommandOptions, diff --git a/app/buck2_client/src/commands/rage/materializer.rs b/app/buck2_client/src/commands/rage/materializer.rs index a90eb8f13317c..fdc7b5bbca3f9 100644 --- a/app/buck2_client/src/commands/rage/materializer.rs +++ b/app/buck2_client/src/commands/rage/materializer.rs @@ -18,6 +18,7 @@ use buck2_client_ctx::events_ctx::PartialResultCtx; use buck2_client_ctx::events_ctx::PartialResultHandler; use buck2_client_ctx::manifold::ManifoldClient; use buck2_client_ctx::subscribers::subscriber::EventSubscriber; +use buck2_client_ctx::subscribers::subscribers::EventSubscribers; use futures::future::BoxFuture; use futures::future::Shared; @@ -31,9 +32,10 @@ pub async fn upload_materializer_data( manifold_id: &String, materializer_data: MaterializerRageUploadData, ) -> anyhow::Result { - let mut buckd = buckd - .await? - .with_subscribers(vec![Box::new(TracingSubscriber) as _]); + let mut buckd = + buckd.await?.with_subscribers(EventSubscribers::new( + vec![Box::new(TracingSubscriber) as _], + )); let mut capture = CaptureStdout::new(); @@ -104,7 +106,7 @@ impl EventSubscriber for TracingSubscriber { Ok(()) } - async fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, error: &buck2_error::Error) -> anyhow::Result<()> { tracing::info!("{:#}", error); Ok(()) } diff --git a/app/buck2_client/src/commands/run.rs b/app/buck2_client/src/commands/run.rs index 1b671d8c8f26a..9cad9eee12ba1 100644 --- a/app/buck2_client/src/commands/run.rs +++ b/app/buck2_client/src/commands/run.rs @@ -181,7 +181,10 @@ impl StreamingCommand for RunCommand { if self.emit_shell { if cfg!(unix) { - buck2_client_ctx::println!("{}", shlex::join(run_args.iter().map(|a| a.as_str())))?; + buck2_client_ctx::println!( + "{}", + shlex::try_join(run_args.iter().map(|a| a.as_str()))? + )?; return ExitResult::success(); } else { return ExitResult::err(RunCommandError::EmitShellNotSupportedOnWindows.into()); diff --git a/app/buck2_client/src/commands/status.rs b/app/buck2_client/src/commands/status.rs index f29c7c466f42d..b0f36c257a647 100644 --- a/app/buck2_client/src/commands/status.rs +++ b/app/buck2_client/src/commands/status.rs @@ -15,6 +15,7 @@ use buck2_client_ctx::client_ctx::ClientCommandContext; use buck2_client_ctx::daemon::client::connect::establish_connection_existing; use buck2_client_ctx::daemon::client::connect::BuckdConnectOptions; use buck2_client_ctx::subscribers::stdout_stderr_forwarder::StdoutStderrForwarder; +use buck2_client_ctx::subscribers::subscribers::EventSubscribers; use buck2_common::argv::Argv; use buck2_common::argv::SanitizedArgv; use buck2_common::daemon_dir::DaemonDir; @@ -66,7 +67,9 @@ impl StatusCommand { if let Ok(bootstrap_client) = establish_connection_existing(&dir).await { statuses.push(process_status( bootstrap_client - .with_subscribers(vec![Box::new(StdoutStderrForwarder)]) + .with_subscribers(EventSubscribers::new(vec![Box::new( + StdoutStderrForwarder, + )])) .with_flushing() .status(self.snapshot) .await?, @@ -139,6 +142,7 @@ fn process_status(status: StatusResponse) -> anyhow::Result { "isolation_dir": status.isolation_dir, "forkserver_pid": serde_json::to_value(status.forkserver_pid)?, "supports_vpnless": status.supports_vpnless.unwrap_or_default(), + "http2": status.http2, })) } diff --git a/app/buck2_client/src/commands/test.rs b/app/buck2_client/src/commands/test.rs index a4ec02ea8c5c0..8605a21243107 100644 --- a/app/buck2_client/src/commands/test.rs +++ b/app/buck2_client/src/commands/test.rs @@ -123,6 +123,25 @@ If include patterns are present, regardless of whether exclude patterns are pres #[clap(long, group = "re_options", alias = "unstable-force-tests-on-re")] unstable_allow_all_tests_on_re: bool, + // NOTE: the field below is given a different name from the test runner's `timeout` to avoid + // confusion between the two parameters. + /// How long to execute tests for. If the timeout is exceeded, Buck2 will exit + /// as quickly as possible and not run further tests. In-flight tests will be + /// cancelled. The test orchestrator will be allowed to shut down gracefully. + /// + /// The exit code is controlled by the test orchestrator (which normally should report zero for + /// this). + /// + /// The format is a concatenation of time spans (separated by spaces). Each time span is an + /// integer number and a suffix. + /// + /// Relevant supported suffixes: seconds, second, sec, s, minutes, minute, min, m, hours, hour, + /// hr, h + /// + /// For example: `5m 10s`, `500s`. + #[clap(long = "overall-timeout")] + timeout: Option, + #[clap(name = "TARGET_PATTERNS", help = "Patterns to test")] patterns: Vec, @@ -191,6 +210,14 @@ impl StreamingCommand for TestCommand { force_use_project_relative_paths: self.unstable_allow_all_tests_on_re, force_run_from_project_root: self.unstable_allow_all_tests_on_re, }), + timeout: self + .timeout + .map(|t| { + let t: std::time::Duration = t.into(); + t.try_into() + }) + .transpose() + .context("Invalid `timeout`")?, }, ctx.stdin() .console_interaction_stream(&self.common_opts.console_opts), @@ -214,8 +241,19 @@ impl StreamingCommand for TestCommand { let console = self.common_opts.console_opts.final_console(); print_build_result(&console, &response.errors)?; - if !response.errors.is_empty() { - console.print_error(&format!("{} BUILDS FAILED", response.errors.len()))?; + + // Filtering out individual types might not be best here. While we just have 1 non-build + // error that seems OK, but if we add more we should reconsider (we could add a type on all + // the build errors, but that seems potentially confusing if we only do that in the test + // command). + let build_errors = response + .errors + .iter() + .filter(|e| e.typ != Some(buck2_data::error::ErrorType::UserDeadlineExpired as _)) + .collect::>(); + + if !build_errors.is_empty() { + console.print_error(&format!("{} BUILDS FAILED", build_errors.len()))?; } // TODO(nmj): Might make sense for us to expose the event ctx, and use its @@ -238,7 +276,7 @@ impl StreamingCommand for TestCommand { line.push(column.to_span_from_test_statuses(statuses)?); line.push(Span::new_unstyled_lossy(". ")); } - line.push(span_from_build_failure_count(response.errors.len())?); + line.push(span_from_build_failure_count(build_errors.len())?); eprint_line(&line)?; print_error_counter(&console, listing_failed, "LISTINGS FAILED", "⚠")?; @@ -263,10 +301,16 @@ impl StreamingCommand for TestCommand { _ => {} } - let exit_result = if let Some(exit_code) = response.exit_code { + let exit_result = if !build_errors.is_empty() { + // If we had build errors, those take precedence and we return their exit code. + ExitResult::from_errors(build_errors.iter().copied()) + } else if let Some(exit_code) = response.exit_code { + // Otherwise, use the exit code from Tpx. ExitResult::status_extended(exit_code) } else { - ExitResult::from_errors(&response.errors) + // But if we had no build errors, and Tpx did not provide an exit code, then that's + // going to be an error. + ExitResult::bail("Test executor did not provide an exit code") }; match self.test_executor_stdout { diff --git a/app/buck2_client_ctx/src/client_ctx.rs b/app/buck2_client_ctx/src/client_ctx.rs index 81a2a982228c6..117bb30fada01 100644 --- a/app/buck2_client_ctx/src/client_ctx.rs +++ b/app/buck2_client_ctx/src/client_ctx.rs @@ -118,6 +118,7 @@ impl<'a> ClientCommandContext<'a> { let config_opts = cmd.common_opts(); Ok(ClientContext { config_overrides: config_opts.config_overrides(arg_matches)?, + cli_modifiers: config_opts.cli_modifiers.clone(), target_platform: config_opts.target_platforms.clone().unwrap_or_default(), host_platform: match config_opts.host_platform_override() { HostPlatformOverride::Default => GrpcHostPlatformOverride::DefaultPlatform, @@ -169,6 +170,7 @@ impl<'a> ClientCommandContext<'a> { .context(CurrentDirIsNotUtf8)? .to_owned(), config_overrides: Default::default(), + cli_modifiers: Default::default(), target_platform: Default::default(), host_platform: Default::default(), host_arch: Default::default(), diff --git a/app/buck2_client_ctx/src/common.rs b/app/buck2_client_ctx/src/common.rs index f2e9f34f3c0eb..7ed2e6cf323c6 100644 --- a/app/buck2_client_ctx/src/common.rs +++ b/app/buck2_client_ctx/src/common.rs @@ -22,6 +22,7 @@ //! } //! ``` use std::path::Path; +use std::str::FromStr; use buck2_cli_proto::common_build_options::ExecutionStrategy; use buck2_cli_proto::config_override::ConfigType; @@ -31,6 +32,7 @@ use clap::ArgGroup; use dupe::Dupe; use gazebo::prelude::*; use termwiz::istty::IsTty; +use tracing::warn; use crate::final_console::FinalConsole; use crate::path_arg::PathArg; @@ -173,6 +175,19 @@ pub struct CommonBuildConfigurationOptions { )] pub target_platforms: Option, + #[clap( + value_name = "VALUE", + long = "modifier", + use_value_delimiter = true, + value_delimiter=',', + short = 'm', + help = "A configuration modifier to configure all targets on the command line. This may be a constraint value target.", + // Needs to be explicitly set, otherwise will treat `-c a b c` -> [a, b, c] + // rather than [a] and other positional arguments `b c`. + number_of_values = 1 + )] + pub cli_modifiers: Vec, + #[clap(long, ignore_case = true, value_name = "HOST", arg_enum)] fake_host: Option, @@ -319,6 +334,7 @@ impl CommonBuildConfigurationOptions { static DEFAULT: CommonBuildConfigurationOptions = CommonBuildConfigurationOptions { config_values: vec![], config_files: vec![], + cli_modifiers: vec![], target_platforms: None, fake_host: None, fake_arch: None, @@ -334,6 +350,28 @@ impl CommonBuildConfigurationOptions { } } +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct BuildReportOption { + /// Fill out the failures in build report as it was done by default in buck1. + fill_out_failures: bool, +} + +impl FromStr for BuildReportOption { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let mut fill_out_failures = false; + if s.to_lowercase() == "fill-out-failures" { + fill_out_failures = true; + } else { + warn!( + "Incorrect syntax for build report option. Got: `{}` but expected `fill-out-failures`", + s.to_owned() + ) + } + Ok(BuildReportOption { fill_out_failures }) + } +} + /// Defines common options for build-like commands (build, test, install). #[allow(rustdoc::invalid_html_tags)] #[derive(Debug, clap::Parser, serde::Serialize, serde::Deserialize)] @@ -345,6 +383,14 @@ pub struct CommonBuildOptions { #[clap(long = "build-report", value_name = "PATH")] build_report: Option, + #[clap( + long = "build-report-options", + requires = "build-report", + value_delimiter = ',', + help = "Comma separated list of build report options (currently only supports a single option: fill-out-failures)." + )] + build_report_options: Vec, + /// Deprecated. Use --build-report=- // TODO(cjhopman): this is probably only used by the e2e framework. remove it from there #[clap(long = "print-build-report", hidden = true)] @@ -452,6 +498,10 @@ impl CommonBuildOptions { pub fn to_proto(&self) -> buck2_cli_proto::CommonBuildOptions { let (unstable_print_build_report, unstable_build_report_filename) = self.build_report(); + let unstable_include_failures_build_report = self + .build_report_options + .iter() + .any(|option| option.fill_out_failures); let concurrency = self .num_threads .map(|num| buck2_cli_proto::Concurrency { concurrency: num }); @@ -482,6 +532,7 @@ impl CommonBuildOptions { skip_missing_targets: self.skip_missing_targets, skip_incompatible_targets: self.skip_incompatible_targets, materialize_failed_inputs: self.materialize_failed_inputs, + unstable_include_failures_build_report, } } } @@ -674,3 +725,69 @@ impl CommonOutputOptions { self.show_full_output || self.show_full_simple_output || self.show_full_json_output } } + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + use clap::Parser; + + use super::*; + + fn parse(args: &[&str]) -> anyhow::Result { + Ok(CommonBuildConfigurationOptions::from_iter_safe( + std::iter::once("program").chain(args.iter().copied()), + )?) + } + + #[test] + fn short_opt_multiple() -> anyhow::Result<()> { + let opts = parse(&["-m", "value1", "-m", "value2"])?; + + assert_eq!(opts.cli_modifiers, vec!["value1", "value2"]); + + Ok(()) + } + + #[test] + fn short_opt_comma_separated() -> anyhow::Result<()> { + let opts = parse(&["-m", "value1,value2"])?; + + assert_eq!(opts.cli_modifiers, vec!["value1", "value2"]); + + Ok(()) + } + + #[test] + fn long_opt_multiple() -> anyhow::Result<()> { + let opts = parse(&["--modifier", "value1", "--modifier", "value2"])?; + + assert_eq!(opts.cli_modifiers, vec!["value1", "value2"]); + + Ok(()) + } + + #[test] + fn long_opt_comma_separated() -> anyhow::Result<()> { + let opts = parse(&["--modifier", "value1,value2"])?; + + assert_eq!(opts.cli_modifiers, vec!["value1", "value2"]); + + Ok(()) + } + + #[test] + fn comma_separated_and_multiple() -> anyhow::Result<()> { + let opts = parse(&["--modifier", "value1,value2", "--modifier", "value3"])?; + + assert_eq!(opts.cli_modifiers, vec!["value1", "value2", "value3"]); + + Ok(()) + } + + #[test] + fn space_separated_fails() -> anyhow::Result<()> { + assert_matches!(parse(&["-m", "value1", "value2"]), Err(..)); + + Ok(()) + } +} diff --git a/app/buck2_client_ctx/src/daemon/client/connect.rs b/app/buck2_client_ctx/src/daemon/client/connect.rs index a127afae6696d..4acdf9e63bc53 100644 --- a/app/buck2_client_ctx/src/daemon/client/connect.rs +++ b/app/buck2_client_ctx/src/daemon/client/connect.rs @@ -29,7 +29,6 @@ use buck2_util::process::async_background_command; use buck2_util::truncate::truncate; use dupe::Dupe; use futures::future::try_join3; -use thiserror::Error; use tokio::io::AsyncReadExt; use tokio::time::timeout; use tonic::codegen::InterceptedService; @@ -49,13 +48,16 @@ use crate::daemon_constraints; use crate::events_ctx::EventsCtx; use crate::immediate_config::ImmediateConfigContext; use crate::startup_deadline::StartupDeadline; +use crate::subscribers::classify_server_stderr::classify_server_stderr; use crate::subscribers::stdout_stderr_forwarder::StdoutStderrForwarder; -use crate::subscribers::subscriber::EventSubscriber; +use crate::subscribers::subscribers::EventSubscribers; /// The client side matcher for DaemonConstraints. #[derive(Clone, Debug)] pub struct DaemonConstraintsRequest { + /// The version of buck2. version: String, + /// Sandcastle id. user_version: Option, desired_trace_io_state: DesiredTraceIoState, pub reject_daemon: Option, @@ -63,6 +65,47 @@ pub struct DaemonConstraintsRequest { pub daemon_startup_config: DaemonStartupConfig, } +#[derive(Debug, derive_more::Display)] +pub(crate) enum ConstraintUnsatisfiedReason { + #[display(fmt = "Version mismatch")] + Version, + #[display(fmt = "User version mismatch")] + UserVersion, + #[display(fmt = "Startup config mismatch")] + StartupConfig, + #[display(fmt = "Reject daemon id")] + RejectDaemonId, + #[display(fmt = "Trace IO mismatch")] + TraceIo, + #[display(fmt = "Materializer state identity mismatch")] + MaterializerStateIdentity, +} + +impl ConstraintUnsatisfiedReason { + pub(crate) fn to_daemon_was_started_reason(&self) -> buck2_data::DaemonWasStartedReason { + match self { + ConstraintUnsatisfiedReason::Version => { + buck2_data::DaemonWasStartedReason::ConstraintMismatchVersion + } + ConstraintUnsatisfiedReason::UserVersion => { + buck2_data::DaemonWasStartedReason::ConstraintMismatchUserVersion + } + ConstraintUnsatisfiedReason::StartupConfig => { + buck2_data::DaemonWasStartedReason::ConstraintMismatchStartupConfig + } + ConstraintUnsatisfiedReason::RejectDaemonId => { + buck2_data::DaemonWasStartedReason::ConstraintRejectDaemonId + } + ConstraintUnsatisfiedReason::TraceIo => { + buck2_data::DaemonWasStartedReason::ConstraintMismatchTraceIo + } + ConstraintUnsatisfiedReason::MaterializerStateIdentity => { + buck2_data::DaemonWasStartedReason::ConstraintMismatchMaterializerStateIdentity + } + } + } +} + impl DaemonConstraintsRequest { pub fn new( immediate_config: &ImmediateConfigContext<'_>, @@ -82,13 +125,16 @@ impl DaemonConstraintsRequest { matches!(self.desired_trace_io_state, DesiredTraceIoState::Enabled) } - fn satisfied(&self, daemon: &buck2_cli_proto::DaemonConstraints) -> bool { + fn satisfied( + &self, + daemon: &buck2_cli_proto::DaemonConstraints, + ) -> Result<(), ConstraintUnsatisfiedReason> { if self.version != daemon.version { - return false; + return Err(ConstraintUnsatisfiedReason::Version); } if self.user_version != daemon.user_version { - return false; + return Err(ConstraintUnsatisfiedReason::UserVersion); } let server_daemon_startup_config = daemon.daemon_startup_config.as_ref().and_then(|c| { @@ -100,12 +146,12 @@ impl DaemonConstraintsRequest { }); if Some(&self.daemon_startup_config) != server_daemon_startup_config.as_ref() { - return false; + return Err(ConstraintUnsatisfiedReason::StartupConfig); } if let Some(r) = &self.reject_daemon { if *r == daemon.daemon_id { - return false; + return Err(ConstraintUnsatisfiedReason::RejectDaemonId); } } @@ -114,12 +160,16 @@ impl DaemonConstraintsRequest { let extra = match &daemon.extra { Some(e) => e, - None => return true, + None => return Ok(()), }; match (self.desired_trace_io_state, extra.trace_io_enabled) { - (DesiredTraceIoState::Enabled, false) => return false, - (DesiredTraceIoState::Disabled, true) => return false, + (DesiredTraceIoState::Enabled, false) => { + return Err(ConstraintUnsatisfiedReason::TraceIo); + } + (DesiredTraceIoState::Disabled, true) => { + return Err(ConstraintUnsatisfiedReason::TraceIo); + } _ => {} } @@ -129,11 +179,11 @@ impl DaemonConstraintsRequest { .as_ref() .map_or(false, |i| i == r) { - return false; + return Err(ConstraintUnsatisfiedReason::MaterializerStateIdentity); } } - true + Ok(()) } } @@ -462,6 +512,7 @@ impl BootstrapBuckdClient { pub async fn connect( paths: &InvocationPaths, constraints: BuckdConnectConstraints, + event_subscribers: &mut EventSubscribers<'_>, ) -> anyhow::Result { let daemon_dir = paths.daemon_dir()?; @@ -485,7 +536,7 @@ impl BootstrapBuckdClient { establish_connection_existing(&daemon_dir).await } BuckdConnectConstraints::Constraints(constraints) => { - establish_connection(paths, constraints).await + establish_connection(paths, constraints, event_subscribers).await } } .with_context(|| daemon_connect_error(paths)) @@ -494,7 +545,7 @@ impl BootstrapBuckdClient { pub fn with_subscribers<'a>( self, - subscribers: Vec>, + subscribers: EventSubscribers<'a>, ) -> BuckdClientConnector<'a> { BuckdClientConnector { client: BuckdClient { @@ -533,7 +584,7 @@ pub struct BuckdConnectOptions<'a> { /// Subscribers manage the way that incoming events from the server are handled. /// The client will forward events and stderr/stdout output from the server to each subscriber. /// By default, this list is set to a single subscriber that notifies the user of basic output from the server. - pub(crate) subscribers: Vec>, + pub(crate) subscribers: EventSubscribers<'a>, pub constraints: BuckdConnectConstraints, } @@ -541,23 +592,21 @@ impl<'a> BuckdConnectOptions<'a> { pub fn existing_only_no_console() -> Self { Self { constraints: BuckdConnectConstraints::ExistingOnly, - subscribers: vec![Box::new(StdoutStderrForwarder)], + subscribers: EventSubscribers::new(vec![Box::new(StdoutStderrForwarder)]), } } pub async fn connect( - self, + mut self, paths: &InvocationPaths, ) -> anyhow::Result> { - match BootstrapBuckdClient::connect(paths, self.constraints) + match BootstrapBuckdClient::connect(paths, self.constraints, &mut self.subscribers) .await .map_err(buck2_error::Error::from) { Ok(client) => Ok(client.with_subscribers(self.subscribers)), Err(e) => { - self.subscribers - .into_iter() - .for_each(|mut s| s.handle_daemon_connection_failure(&e)); + self.subscribers.handle_daemon_connection_failure(&e); Err(e.into()) } } @@ -585,6 +634,7 @@ pub async fn establish_connection_existing( async fn establish_connection( paths: &InvocationPaths, constraints: DaemonConstraintsRequest, + event_subscribers: &mut EventSubscribers<'_>, ) -> anyhow::Result { // There are many places where `establish_connection_inner` may hang. // If it does, better print something to the user instead of hanging quietly forever. @@ -593,7 +643,7 @@ async fn establish_connection( deadline .down( "establishing connection to Buck daemon or start a daemon", - |timeout| establish_connection_inner(paths, constraints, timeout), + |timeout| establish_connection_inner(paths, constraints, timeout, event_subscribers), ) .await } @@ -602,6 +652,7 @@ async fn establish_connection_inner( paths: &InvocationPaths, constraints: DaemonConstraintsRequest, deadline: StartupDeadline, + event_subscribers: &mut EventSubscribers<'_>, ) -> anyhow::Result { let daemon_dir = paths.daemon_dir()?; let connect_before_restart = deadline @@ -625,24 +676,34 @@ async fn establish_connection_inner( // Even if we didn't connect before, it's possible that we just raced with another invocation // starting the server, so we try to connect again while holding the lock. - if let Ok(channel) = try_connect_existing(&daemon_dir, &deadline).await { - let mut client = channel.upgrade().await?; - if constraints.satisfied(&client.constraints) { - return Ok(client); - } - deadline - .run( - "sending kill command to the Buck daemon", - client.kill_for_constraints_mismatch(), - ) - .await?; - } + let daemon_was_started_reason = + match try_connect_existing(&daemon_dir, &deadline, &lifecycle_lock).await { + Ok(channel) => { + let mut client = channel.upgrade().await?; + let daemon_was_started_reason = match constraints.satisfied(&client.constraints) { + Ok(()) => return Ok(client), + Err(reason) => reason.to_daemon_was_started_reason(), + }; + deadline + .run( + "sending kill command to the Buck daemon", + client.kill_for_constraints_mismatch(), + ) + .await?; + daemon_was_started_reason + } + Err(..) => buck2_data::DaemonWasStartedReason::CouldNotConnectToDaemon, + }; // Daemon dir may be corrupted. Safer to delete it. lifecycle_lock .clean_daemon_dir() .context("Cleaning daemon dir")?; + event_subscribers + .eprintln("Could not connect to buck2 daemon, starting a new one...") + .await?; + // Now there's definitely no server that can be connected to // TODO(cjhopman): a non-responsive buckd process may be somehow lingering around and we should probably kill it off here. lifecycle_lock.start_server().await?; @@ -660,14 +721,21 @@ async fn establish_connection_inner( let client = channel.upgrade().await?; - if !constraints.satisfied(&client.constraints) { + if let Err(reason) = constraints.satisfied(&client.constraints) { return Err(BuckdConnectError::BuckDaemonConstraintWrongAfterStart { + reason, expected: constraints.clone(), actual: client.constraints, } .into()); } + event_subscribers.handle_daemon_started(daemon_was_started_reason); + + event_subscribers + .eprintln("Connected to new buck2 daemon.") + .await?; + Ok(client) } @@ -690,7 +758,7 @@ async fn try_connect_existing_before_daemon_restart( match BuckdProcessInfo::load_and_create_channel(daemon_dir).await { Ok(channel) => { let client = channel.upgrade().await?; - if constraints.satisfied(&client.constraints) { + if constraints.satisfied(&client.constraints).is_ok() { Ok(ConnectBeforeRestart::Accepted(client)) } else { Ok(ConnectBeforeRestart::Rejected) @@ -706,6 +774,7 @@ async fn try_connect_existing_before_daemon_restart( async fn try_connect_existing( daemon_dir: &DaemonDir, timeout: &StartupDeadline, + _lock: &BuckdLifecycle<'_>, ) -> anyhow::Result { timeout .min(buckd_startup_timeout()?)? @@ -772,8 +841,8 @@ async fn get_constraints( ) -> anyhow::Result { // NOTE: No tailers in bootstrap client, we capture logs if we fail to connect, but // otherwise we leave them alone. - let status = EventsCtx::new(vec![Box::new(StdoutStderrForwarder)]) - .unpack_oneshot(&mut None, || { + let status = EventsCtx::new(EventSubscribers::new(vec![Box::new(StdoutStderrForwarder)])) + .unpack_oneshot(None, { client.status(tonic::Request::new(buck2_cli_proto::StatusRequest { snapshot: false, })) @@ -790,8 +859,9 @@ async fn get_constraints( Ok(status.daemon_constraints.unwrap_or_default()) } -#[derive(Debug, Error)] +#[derive(Debug, buck2_error::Error)] #[allow(clippy::large_enum_variant)] +#[buck2(tag = DaemonConnect)] enum BuckdConnectError { #[error( "buck daemon startup failed with exit code {code}\nstdout:\n{stdout}\nstderr:\n{stderr}" @@ -802,13 +872,15 @@ enum BuckdConnectError { stderr: String, }, #[error( - "during buck daemon startup, the started process did not match constraints.\nexpected: {expected:?}\nactual: {actual:?}" + "during buck daemon startup, the started process did not match constraints ({reason}).\nexpected: {expected:?}\nactual: {actual:?}" )] BuckDaemonConstraintWrongAfterStart { + reason: ConstraintUnsatisfiedReason, expected: DaemonConstraintsRequest, actual: buck2_cli_proto::DaemonConstraints, }, #[error("Error connecting to the daemon, daemon stderr follows:\n{stderr}")] + #[buck2(tag = Some(classify_server_stderr(stderr)))] ConnectError { stderr: String }, } @@ -860,21 +932,21 @@ mod tests { fn test_constraints_equal_for_same_constraints() { let req = request(DesiredTraceIoState::Enabled); let daemon = constraints(true); - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); } #[test] fn test_constraints_equal_for_trace_io_existing() { let req = request(DesiredTraceIoState::Existing); let daemon = constraints(true); - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); } #[test] fn test_constraints_unequal_for_trace_io() { let req = request(DesiredTraceIoState::Disabled); let daemon = constraints(true); - assert!(!req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_err()); } #[test] @@ -907,11 +979,11 @@ mod tests { ), }; - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); req.reject_daemon = Some("zzz".to_owned()); - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); req.reject_daemon = Some("ddd".to_owned()); - assert!(!req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_err()); } #[test] @@ -938,11 +1010,11 @@ mod tests { ), }; - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); req.reject_materializer_state = Some("zzz".to_owned()); - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); req.reject_materializer_state = Some("mmm".to_owned()); - assert!(!req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_err()); } #[test] @@ -969,8 +1041,8 @@ mod tests { ), }; - assert!(req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_ok()); req.daemon_startup_config.daemon_buster = Some("1".to_owned()); - assert!(!req.satisfied(&daemon)); + assert!(req.satisfied(&daemon).is_err()); } } diff --git a/app/buck2_client_ctx/src/daemon/client/kill.rs b/app/buck2_client_ctx/src/daemon/client/kill.rs index bd38e8305cd78..d3fd500350e02 100644 --- a/app/buck2_client_ctx/src/daemon/client/kill.rs +++ b/app/buck2_client_ctx/src/daemon/client/kill.rs @@ -186,7 +186,8 @@ fn get_callers_for_kill() -> Vec { ) -> Option<(Pid, Duration)> { // Specifics about this process need to be refreshed by this time. let proc = system.process(pid)?; - let title = shlex::join(proc.cmd().iter().map(|s| s.as_str())); + let title = + shlex::try_join(proc.cmd().iter().map(|s| s.as_str())).expect("Null byte unexpected"); process_tree.push(title); let parent_pid = proc.parent()?; system.refresh_process_specifics(parent_pid, ProcessRefreshKind::new()); diff --git a/app/buck2_client_ctx/src/daemon/client/mod.rs b/app/buck2_client_ctx/src/daemon/client/mod.rs index 7e873b33b1cd6..fceb4be32f32f 100644 --- a/app/buck2_client_ctx/src/daemon/client/mod.rs +++ b/app/buck2_client_ctx/src/daemon/client/mod.rs @@ -9,6 +9,7 @@ use std::fs::create_dir_all; use std::fs::File; +use std::mem; use std::time::Duration; use anyhow::Context; @@ -20,6 +21,7 @@ use buck2_cli_proto::*; use buck2_common::daemon_dir::DaemonDir; use buck2_core::fs::fs_util; use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; +use buck2_data::error::ErrorTag; use buck2_event_log::stream_value::StreamValue; use fs4::FileExt; use futures::future::BoxFuture; @@ -71,11 +73,7 @@ impl<'a> BuckdClientConnector<'a> { } pub fn error_observers(&self) -> impl Iterator { - self.client - .events_ctx - .subscribers - .iter() - .filter_map(|s| s.as_error_observer()) + self.client.events_ctx.subscribers.error_observers() } } @@ -159,6 +157,28 @@ enum GrpcToStreamError { EmptyCommandProgress, } +/// Convert tonic error to our error. +/// +/// This function **must** be used explicitly to convert the error, because we want a tag. +pub(crate) fn tonic_status_to_error(status: tonic::Status) -> anyhow::Error { + let mut tags = vec![ErrorTag::ClientGrpc]; + if status.code() == tonic::Code::ResourceExhausted { + // The error looks like this: + // ``` + // status: ResourceExhausted + // message: "Cannot return body with more than 4GB of data but got 4294992775 bytes" + // details: [], metadata: MetadataMap { headers: {} } + // ``` + if status + .message() + .contains("Cannot return body with more than") + { + tags.push(ErrorTag::GrpcResponseMessageTooLarge); + } + } + buck2_error::Error::from(status).tag(tags).into() +} + /// Translates a tonic streaming response into a stream of StreamValues, the set of things that can flow across the gRPC /// event stream. fn grpc_to_stream( @@ -170,6 +190,7 @@ fn grpc_to_stream( }; let stream = stream + .map_err(tonic_status_to_error) .map_ok(|e| stream::iter(e.messages.into_iter().map(anyhow::Ok))) .try_flatten(); @@ -227,6 +248,7 @@ impl<'a> BuckdClient<'a> { let response = command(client, Request::new(request)) .await + .map_err(tonic_status_to_error) .context("Error dispatching request"); let stream = grpc_to_stream(response); pin_mut!(stream); @@ -244,7 +266,7 @@ impl<'a> BuckdClient<'a> { let outcome = self .events_ctx // Safe to unwrap tailers here because they are instantiated prior to a command being called. - .unpack_oneshot(&mut self.tailers, || { + .unpack_oneshot(mem::take(&mut self.tailers), { self.client.status(Request::new(StatusRequest { snapshot })) }) .await; @@ -277,7 +299,10 @@ impl<'a, 'b> FlushingBuckdClient<'a, 'b> { } async fn exit(&mut self) -> anyhow::Result<()> { - self.inner.events_ctx.flush(&mut self.inner.tailers).await?; + self.inner + .events_ctx + .flush(mem::take(&mut self.inner.tailers)) + .await?; Ok(()) } @@ -398,7 +423,7 @@ macro_rules! oneshot_method { let res = self .inner .events_ctx - .unpack_oneshot(&mut self.inner.tailers, || { + .unpack_oneshot(mem::take(&mut self.inner.tailers), { self.inner.client.$method(Request::new(req)) }) .await; @@ -541,7 +566,7 @@ impl<'a, 'b> FlushingBuckdClient<'a, 'b> { oneshot_method!(flush_dep_files, FlushDepFilesRequest, GenericResponse); - debug_method!(unstable_crash, UnstableCrashRequest, UnstableCrashResponse); + oneshot_method!(unstable_crash, UnstableCrashRequest, GenericResponse); debug_method!(segfault, SegfaultRequest, SegfaultResponse); debug_method!( unstable_heap_dump, diff --git a/app/buck2_client_ctx/src/events_ctx.rs b/app/buck2_client_ctx/src/events_ctx.rs index 6f2e8e44e562a..b1ccfe22b8129 100644 --- a/app/buck2_client_ctx/src/events_ctx.rs +++ b/app/buck2_client_ctx/src/events_ctx.rs @@ -8,6 +8,7 @@ */ use std::ops::ControlFlow; +use std::pin::pin; use std::sync::Arc; use std::time::SystemTime; @@ -18,8 +19,9 @@ use buck2_cli_proto::CommandResult; use buck2_common::daemon_dir::DaemonDir; use buck2_event_log::stream_value::StreamValue; use buck2_events::BuckEvent; -use futures::stream::FuturesUnordered; +use futures::stream; use futures::Future; +use futures::FutureExt; use futures::Stream; use futures::StreamExt; use gazebo::prelude::VecExt; @@ -32,11 +34,13 @@ use crate::command_outcome::CommandOutcome; use crate::console_interaction_stream::ConsoleInteraction; use crate::console_interaction_stream::ConsoleInteractionStream; use crate::console_interaction_stream::NoopConsoleInteraction; +use crate::daemon::client::tonic_status_to_error; +use crate::daemon::client::NoPartialResultHandler; use crate::exit_result::ExitResult; use crate::file_tailer::FileTailer; use crate::file_tailer::StdoutOrStderr; -use crate::subscribers::subscriber::EventSubscriber; use crate::subscribers::subscriber::Tick; +use crate::subscribers::subscribers::EventSubscribers; use crate::ticker::Ticker; /// Target number of self.tick() calls per second. These can be used by implementations for regular updates, for example @@ -91,7 +95,8 @@ pub struct PartialResultCtx<'a, 'b> { impl<'a, 'b> PartialResultCtx<'a, 'b> { pub async fn stdout(&mut self, bytes: &[u8]) -> anyhow::Result<()> { self.inner - .handle_subscribers(|subscriber| subscriber.handle_output(bytes)) + .subscribers + .for_each_subscriber(|subscriber| subscriber.handle_output(bytes)) .await } } @@ -99,7 +104,7 @@ impl<'a, 'b> PartialResultCtx<'a, 'b> { /// Manages incoming event streams from the daemon for the buck2 client and /// forwards them to the appropriate subscribers registered on this struct pub struct EventsCtx<'a> { - pub(crate) subscribers: Vec>, + pub(crate) subscribers: EventSubscribers<'a>, ticker: Ticker, client_cpu_tracker: ClientCpuTracker, } @@ -117,7 +122,7 @@ pub struct FileTailers { } impl<'a> EventsCtx<'a> { - pub fn new(subscribers: Vec>) -> Self { + pub fn new(subscribers: EventSubscribers<'a>) -> Self { Self { subscribers, ticker: Ticker::new(TICKS_PER_SECOND), @@ -172,8 +177,12 @@ impl<'a> EventsCtx<'a> { async fn dispatch_tailer_event(&mut self, event: FileTailerEvent) -> anyhow::Result<()> { match event { - FileTailerEvent::Stdout(stdout) => self.handle_tailer_stdout(&stdout).await, - FileTailerEvent::Stderr(stderr) => self.handle_tailer_stderr(&stderr).await, + FileTailerEvent::Stdout(out) | FileTailerEvent::Stderr(out) => { + // Sending daemon stdout to stderr. + // Daemon is not supposed to write anything to stdout. + // But if daemon does, it should not be used as standard output of buck2. + self.handle_tailer_stderr(&out).await + } } } @@ -185,7 +194,7 @@ impl<'a> EventsCtx<'a> { mut console_interaction: Option>, ) -> anyhow::Result where - S: Stream> + Unpin, + S: Stream>, Handler: PartialResultHandler, { let mut noop_console_interaction = NoopConsoleInteraction; @@ -204,7 +213,8 @@ impl<'a> EventsCtx<'a> { let mut tailers = tailers.unwrap_or_else(FileTailers::empty); - let mut stream = stream.ready_chunks(1000); + let stream = stream.ready_chunks(1000); + let mut stream = pin!(stream); // NOTE: When unpacking the stream we capture any shutdown event we encounter. If we fail // to unpack the stream to completion, we'll use that later. @@ -240,8 +250,8 @@ impl<'a> EventsCtx<'a> { } }; - let flush_result = self.flush(&mut Some(tailers)).await; - let exit_result = self.handle_exit().await; + let flush_result = self.flush(Some(tailers)).await; + let exit_result = self.subscribers.handle_exit().await; let command_result = match (command_result, shutdown) { (Ok(r), _) => r, @@ -279,7 +289,7 @@ impl<'a> EventsCtx<'a> { console_interaction: Option>, ) -> anyhow::Result> where - S: Stream> + Unpin, + S: Stream>, Res: TryFrom, Handler: PartialResultHandler, { @@ -293,16 +303,6 @@ impl<'a> EventsCtx<'a> { } } - pub async fn flushing_tailers>( - &mut self, - tailers: &mut Option, - f: impl FnOnce() -> Fut, - ) -> anyhow::Result { - let res = f().await; - self.flush(tailers).await?; - Ok(res) - } - /// Unpack a single `CommandResult`, log any failures if necessary, and convert it to a /// `CommandOutcome` pub async fn unpack_oneshot< @@ -310,51 +310,37 @@ impl<'a> EventsCtx<'a> { Fut: Future, tonic::Status>>, >( &mut self, - tailers: &mut Option, - f: impl FnOnce() -> Fut, + tailers: Option, + f: Fut, ) -> anyhow::Result> { - let res = self.flushing_tailers(tailers, f).await?; - // important - do not early return before flushing the buffers! - let inner = res?.into_inner(); - self.handle_command_result(&inner).await?; - - convert_result(inner) - } - - /// Helper method to abstract the process of applying an `EventSubscriber` method to all of the subscribers. - /// Quits on the first error encountered. - async fn handle_subscribers<'b, Fut>( - &'b mut self, - f: impl FnMut(&'b mut Box) -> Fut, - ) -> anyhow::Result<()> - where - Fut: Future> + 'b, - { - let mut futures: FuturesUnordered<_> = self.subscribers.iter_mut().map(f).collect(); - while let Some(res) = futures.next().await { - res?; - } - Ok(()) + let stream = stream::once(f.map(|result| { + result + .map(|command_result| StreamValue::Result(Box::new(command_result.into_inner()))) + .map_err(tonic_status_to_error) + })); + self.unpack_stream(&mut NoPartialResultHandler, stream, tailers, None) + .await } async fn handle_error_owned(&mut self, error: anyhow::Error) -> anyhow::Error { + let error: buck2_error::Error = error.into(); let result = self - .handle_subscribers(|subscriber| subscriber.handle_error(&error)) + .subscribers + .for_each_subscriber(|subscriber| subscriber.handle_error(&error)) .await; match result { - Ok(()) => error, + Ok(()) => error.into(), Err(e) => EventsCtxError::WrappedStreamError { - source: error, + source: error.into(), other: e, } .into(), } } - pub async fn flush(&mut self, tailers: &mut Option) -> anyhow::Result<()> { - let tailers = match tailers.take() { - Some(tailers) => tailers, - None => return Ok(()), + pub async fn flush(&mut self, tailers: Option) -> anyhow::Result<()> { + let Some(tailers) = tailers else { + return Ok(()); }; let mut streams = tailers.stop_reading(); @@ -398,20 +384,17 @@ fn convert_result EventsCtx<'a> { - async fn handle_tailer_stdout(&mut self, raw_output: &[u8]) -> anyhow::Result<()> { - self.handle_subscribers(|subscriber| subscriber.handle_output(raw_output)) - .await - } - async fn handle_tailer_stderr(&mut self, stderr: &[u8]) -> anyhow::Result<()> { let stderr = String::from_utf8_lossy(stderr); let stderr = stderr.trim_end(); - self.handle_subscribers(|subscriber| subscriber.handle_tailer_stderr(stderr)) + self.subscribers + .for_each_subscriber(|subscriber| subscriber.handle_tailer_stderr(stderr)) .await } async fn handle_console_interaction(&mut self, c: char) -> anyhow::Result<()> { - self.handle_subscribers(|subscriber| subscriber.handle_console_interaction(c)) + self.subscribers + .for_each_subscriber(|subscriber| subscriber.handle_console_interaction(c)) .await } @@ -446,7 +429,8 @@ impl<'a> EventsCtx<'a> { } Arc::new(event) }); - self.handle_subscribers(|subscriber| subscriber.handle_events(&events)) + self.subscribers + .for_each_subscriber(|subscriber| subscriber.handle_events(&events)) .await } @@ -454,7 +438,8 @@ impl<'a> EventsCtx<'a> { &mut self, result: &buck2_cli_proto::CommandResult, ) -> anyhow::Result<()> { - self.handle_subscribers(|subscriber| subscriber.handle_command_result(result)) + self.subscribers + .for_each_subscriber(|subscriber| subscriber.handle_command_result(result)) .await } @@ -462,22 +447,10 @@ impl<'a> EventsCtx<'a> { /// A subscriber will have the opportunity to do an arbitrary process at a reliable interval. /// In particular, this is crucial for superconsole so that it can draw itself consistently. async fn tick(&mut self, tick: &Tick) -> anyhow::Result<()> { - self.handle_subscribers(|subscriber| subscriber.tick(tick)) + self.subscribers + .for_each_subscriber(|subscriber| subscriber.tick(tick)) .await } - - async fn handle_exit(&mut self) -> anyhow::Result<()> { - let mut r = Ok(()); - for subscriber in &mut self.subscribers { - // Exit all subscribers, do not stop on first one. - let subscriber_err = subscriber.exit().await; - if r.is_ok() { - // Keep first error. - r = subscriber_err; - } - } - r - } } impl FileTailers { diff --git a/app/buck2_client_ctx/src/exit_result.rs b/app/buck2_client_ctx/src/exit_result.rs index 3fcc89f043c1c..9557d8800380a 100644 --- a/app/buck2_client_ctx/src/exit_result.rs +++ b/app/buck2_client_ctx/src/exit_result.rs @@ -142,7 +142,7 @@ impl ExitResult { } } - pub fn from_errors(errors: &[buck2_data::ErrorReport]) -> Self { + pub fn from_errors<'a>(errors: impl IntoIterator) -> Self { let mut has_infra = false; let mut has_user = false; for e in errors { diff --git a/app/buck2_client_ctx/src/file_tailer.rs b/app/buck2_client_ctx/src/file_tailer.rs index 2c0be8d87fb99..32bbc41029271 100644 --- a/app/buck2_client_ctx/src/file_tailer.rs +++ b/app/buck2_client_ctx/src/file_tailer.rs @@ -16,6 +16,7 @@ use std::time::Duration; use anyhow::Context; use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; +use buck2_util::threads::thread_spawn; use futures::FutureExt; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::oneshot; @@ -75,7 +76,7 @@ impl FileTailer { // TODO(cjhopman): It would probably be nicer to implement this via inotify/fsevents/etc // rather than just repeatedly reading the file, but I tried to use each of // https://crates.io/crates/hotwatch and https://crates.io/crates/notify and neither worked. - let thread = std::thread::spawn(move || { + let thread = thread_spawn("buck2-tailer", move || { let runtime = client_tokio_runtime()?; let res = runtime.block_on(async move { let mut interval = tokio::time::interval(Duration::from_millis(200)); @@ -115,7 +116,7 @@ impl FileTailer { }); res.with_context(|| format!("Failed to read from `{}`", file)) - }); + })?; Ok(Self { end_signaller: Some(tx), diff --git a/app/buck2_client_ctx/src/ide_support.rs b/app/buck2_client_ctx/src/ide_support.rs index 6aeb9c5fe1b78..28e45652348c7 100644 --- a/app/buck2_client_ctx/src/ide_support.rs +++ b/app/buck2_client_ctx/src/ide_support.rs @@ -76,7 +76,7 @@ impl Deserialize<'a>> Decoder for LspMessageLikeDecoder { } #[cfg(test)] -mod test { +mod tests { use assert_matches::assert_matches; use lsp_server::Message; use lsp_server::Request; diff --git a/app/buck2_client_ctx/src/lib.rs b/app/buck2_client_ctx/src/lib.rs index 6113cec80729b..2adf20465c0f1 100644 --- a/app/buck2_client_ctx/src/lib.rs +++ b/app/buck2_client_ctx/src/lib.rs @@ -9,11 +9,13 @@ #![feature(once_cell_try)] #![feature(async_closure)] +#![feature(error_generic_member_access)] #![feature(if_let_guard)] #![feature(let_chains)] #![feature(try_blocks)] #![feature(try_trait_v2)] #![feature(used_with_arg)] +#![feature(round_char_boundary)] pub mod build_count; pub mod chunk_reader; diff --git a/app/buck2_client_ctx/src/stdin.rs b/app/buck2_client_ctx/src/stdin.rs index 463b892b0dfab..8c71529f55839 100644 --- a/app/buck2_client_ctx/src/stdin.rs +++ b/app/buck2_client_ctx/src/stdin.rs @@ -15,6 +15,7 @@ use std::task::Poll; use std::thread::JoinHandle; use buck2_core::buck2_env; +use buck2_util::threads::thread_spawn; use bytes::Bytes; use futures::stream::Fuse; use futures::stream::StreamExt; @@ -96,7 +97,7 @@ impl State { buffer_size, mut tx, } => { - let handle = std::thread::spawn({ + let handle = thread_spawn("buck2-stdin", { move || { #[allow(clippy::let_and_return)] let stdin = std::io::stdin().lock(); @@ -109,7 +110,8 @@ impl State { // NOTE: We ignore send errors since there is no point in reading without a receiver. let _ignored = read_and_forward(stdin, &mut tx, buffer_size); } - }); + }) + .unwrap(); Self::Started(handle) } diff --git a/app/buck2_client_ctx/src/streaming.rs b/app/buck2_client_ctx/src/streaming.rs index d55a1ce02e166..71ebc1db31b98 100644 --- a/app/buck2_client_ctx/src/streaming.rs +++ b/app/buck2_client_ctx/src/streaming.rs @@ -35,11 +35,12 @@ use crate::subscribers::get::try_get_event_log_subscriber; use crate::subscribers::get::try_get_re_log_subscriber; use crate::subscribers::recorder::try_get_invocation_recorder; use crate::subscribers::subscriber::EventSubscriber; +use crate::subscribers::subscribers::EventSubscribers; fn default_subscribers<'a, T: StreamingCommand>( cmd: &T, ctx: &ClientCommandContext<'a>, -) -> anyhow::Result>> { +) -> anyhow::Result> { let console_opts = cmd.console_opts(); let mut subscribers = vec![]; let expect_spans = cmd.should_expect_spans(); @@ -81,7 +82,7 @@ fn default_subscribers<'a, T: StreamingCommand>( subscribers.push(recorder); subscribers.extend(cmd.extra_subscribers()); - Ok(subscribers) + Ok(EventSubscribers::new(subscribers)) } /// Trait to generalize the behavior of executable buck2 commands that rely on a server. diff --git a/app/buck2_client_ctx/src/subscribers/classify_server_stderr.rs b/app/buck2_client_ctx/src/subscribers/classify_server_stderr.rs new file mode 100644 index 0000000000000..125c7f8635bb9 --- /dev/null +++ b/app/buck2_client_ctx/src/subscribers/classify_server_stderr.rs @@ -0,0 +1,33 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use buck2_data::error::ErrorTag; + +pub(crate) fn classify_server_stderr(stderr: &str) -> ErrorTag { + if stderr.is_empty() { + ErrorTag::ServerStderrEmpty + } else if stderr.contains(": size mismatch detected") { + // P1181704561 + ErrorTag::ServerJemallocAssert + } else if stderr.contains("panicked at") { + // Sample output of `buck2 debug crash`: P1159041719 + ErrorTag::ServerPanicked + } else if stderr.contains("has overflowed its stack") { + // Stderr looks like this: + // ``` + // thread 'buck2-dm' has overflowed its stack + // ``` + ErrorTag::ServerStackOverflow + } else if stderr.contains("Signal 11 (SIGSEGV)") { + // P1180289404 + ErrorTag::ServerSegv + } else { + ErrorTag::ServerStderrUnknown + } +} diff --git a/app/buck2_client_ctx/src/subscribers/errorconsole.rs b/app/buck2_client_ctx/src/subscribers/errorconsole.rs index 10649913ff792..844754bec28c9 100644 --- a/app/buck2_client_ctx/src/subscribers/errorconsole.rs +++ b/app/buck2_client_ctx/src/subscribers/errorconsole.rs @@ -106,7 +106,7 @@ impl UnpackingEventSubscriber for ErrorConsole { Ok(()) } - async fn handle_error(&mut self, _error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, _error: &buck2_error::Error) -> anyhow::Result<()> { Ok(()) } diff --git a/app/buck2_client_ctx/src/subscribers/event_log.rs b/app/buck2_client_ctx/src/subscribers/event_log.rs index 142007be0c1fe..5fccde65ad93d 100644 --- a/app/buck2_client_ctx/src/subscribers/event_log.rs +++ b/app/buck2_client_ctx/src/subscribers/event_log.rs @@ -62,6 +62,20 @@ impl<'a> EventSubscriber for EventLog<'a> { self.writer.write_events(events).await } + async fn handle_tailer_stderr(&mut self, _stderr: &str) -> anyhow::Result<()> { + // TODO(nga): currently we mostly ignore buckd stderr. + // It is very important to investigate crashes of buckd. + // + // We attach truncated log to Scuba since D53337966 + // (although we probably shouldn't do that). + // + // Regardless of that we should do either or both of the following: + // - write it to event log if it is interesting (e.g. crash) + // - upload it to manifold unconditionally as a separate file + // (but only relevant part, since command start) + Ok(()) + } + async fn handle_command_result( &mut self, result: &buck2_cli_proto::CommandResult, diff --git a/app/buck2_client_ctx/src/subscribers/mod.rs b/app/buck2_client_ctx/src/subscribers/mod.rs index ff917ea34f462..ca6251f2e6c9a 100644 --- a/app/buck2_client_ctx/src/subscribers/mod.rs +++ b/app/buck2_client_ctx/src/subscribers/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod build_graph_stats; pub(crate) mod build_id_writer; +pub(crate) mod classify_server_stderr; pub(crate) mod errorconsole; pub mod event_log; pub mod get; @@ -19,4 +20,6 @@ pub(crate) mod simpleconsole; pub mod stdout_stderr_forwarder; pub mod subscriber; pub mod subscriber_unpack; +#[allow(clippy::module_inception)] +pub mod subscribers; pub mod superconsole; diff --git a/app/buck2_client_ctx/src/subscribers/recorder.rs b/app/buck2_client_ctx/src/subscribers/recorder.rs index da4ddaa771d63..3f7d7be984c7d 100644 --- a/app/buck2_client_ctx/src/subscribers/recorder.rs +++ b/app/buck2_client_ctx/src/subscribers/recorder.rs @@ -7,1148 +7,1249 @@ * of this source tree. */ +use std::cmp; +use std::collections::HashMap; +use std::collections::HashSet; +use std::future::Future; +use std::io::Write; +use std::iter; use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; use std::sync::Arc; - +use std::time::Duration; +use std::time::Instant; +use std::time::SystemTime; + +use anyhow::Context; +use async_trait::async_trait; +use buck2_cli_proto::command_result; +use buck2_common::convert::ProstDurationExt; +use buck2_core::fs::fs_util; +use buck2_core::fs::paths::abs_path::AbsPathBuf; +use buck2_data::error::ErrorTag; +use buck2_error::classify::best_tag; +use buck2_error::classify::ERROR_TAG_UNCLASSIFIED; +use buck2_event_observer::action_stats; +use buck2_event_observer::cache_hit_rate::total_cache_hit_rate; +use buck2_event_observer::last_command_execution_kind; +use buck2_event_observer::last_command_execution_kind::LastCommandExecutionKind; +use buck2_events::errors::create_error_report; +use buck2_events::sink::scribe::new_thrift_scribe_sink_if_enabled; +use buck2_events::BuckEvent; +use buck2_util::cleanup_ctx::AsyncCleanupContext; +use buck2_util::system_stats::system_memory_stats; +use buck2_wrapper_common::invocation_id::TraceId; use dupe::Dupe; +use fbinit::FacebookInit; +use futures::FutureExt; +use gazebo::prelude::VecExt; +use gazebo::variants::VariantName; +use itertools::Itertools; +use termwiz::istty::IsTty; use crate::build_count::BuildCountManager; use crate::client_ctx::ClientCommandContext; use crate::client_metadata::ClientMetadata; use crate::common::CommonDaemonCommandOptions; +use crate::subscribers::classify_server_stderr::classify_server_stderr; +use crate::subscribers::observer::ErrorObserver; +use crate::subscribers::subscriber::EventSubscriber; + +struct ErrorIntermediate { + processed: buck2_data::ProcessedErrorReport, + /// Append stderr to the message before sending the report. + want_stderr: bool, + best_tag: Option, +} + +pub(crate) struct InvocationRecorder<'a> { + fb: FacebookInit, + write_to_path: Option, + command_name: &'static str, + cli_args: Vec, + isolation_dir: String, + start_time: Instant, + async_cleanup_context: AsyncCleanupContext<'a>, + build_count_manager: BuildCountManager, + trace_id: TraceId, + command_end: Option, + command_duration: Option, + re_session_id: Option, + re_experiment_name: Option, + critical_path_duration: Option, + tags: Vec, + run_local_count: u64, + run_remote_count: u64, + run_action_cache_count: u64, + run_remote_dep_file_cache_count: u64, + run_skipped_count: u64, + run_fallback_count: u64, + local_actions_executed_via_worker: u64, + first_snapshot: Option, + last_snapshot: Option, + min_build_count_since_rebase: u64, + cache_upload_count: u64, + cache_upload_attempt_count: u64, + parsed_target_patterns: Option, + filesystem: String, + watchman_version: Option, + eden_version: Option, + test_info: Option, + eligible_for_full_hybrid: bool, + max_event_client_delay: Option, + max_malloc_bytes_active: Option, + max_malloc_bytes_allocated: Option, + run_command_failure_count: u64, + event_count: u64, + time_to_first_action_execution: Option, + materialization_output_size: u64, + initial_materializer_entries_from_sqlite: Option, + time_to_command_start: Option, + time_to_command_critical_section: Option, + time_to_first_analysis: Option, + time_to_load_first_build_file: Option, + time_to_first_command_execution_start: Option, + time_to_first_test_discovery: Option, + system_total_memory_bytes: Option, + file_watcher_stats: Option, + time_to_last_action_execution_end: Option, + initial_sink_success_count: Option, + initial_sink_failure_count: Option, + initial_sink_dropped_count: Option, + sink_max_buffer_depth: u64, + soft_error_categories: HashSet, + concurrent_command_blocking_duration: Option, + metadata: HashMap, + analysis_count: u64, + daemon_in_memory_state_is_corrupted: bool, + daemon_materializer_state_is_corrupted: bool, + enable_restarter: bool, + restarted_trace_id: Option, + has_command_result: bool, + has_end_of_stream: bool, + compressed_event_log_size_bytes: Option>, + critical_path_backend: Option, + instant_command_is_success: Option, + bxl_ensure_artifacts_duration: Option, + initial_re_upload_bytes: Option, + initial_re_download_bytes: Option, + concurrent_command_ids: HashSet, + daemon_connection_failure: bool, + /// Daemon started by this command. + daemon_was_started: Option, + client_metadata: Vec, + errors: Vec, + /// To append to gRPC errors. + server_stderr: String, + target_rule_type_names: Vec, +} -mod imp { - use std::cmp; - use std::collections::HashMap; - use std::collections::HashSet; - use std::future::Future; - use std::io::Write; - use std::sync::atomic::AtomicU64; - use std::sync::atomic::Ordering; - use std::sync::Arc; - use std::time::Duration; - use std::time::Instant; - use std::time::SystemTime; - - use anyhow::Context; - use async_trait::async_trait; - use buck2_cli_proto::command_result; - use buck2_common::convert::ProstDurationExt; - use buck2_core::fs::fs_util; - use buck2_core::fs::paths::abs_path::AbsPathBuf; - use buck2_event_observer::action_stats; - use buck2_event_observer::cache_hit_rate::total_cache_hit_rate; - use buck2_event_observer::last_command_execution_kind; - use buck2_event_observer::last_command_execution_kind::LastCommandExecutionKind; - use buck2_events::errors::create_error_report; - use buck2_events::sink::scribe::new_thrift_scribe_sink_if_enabled; - use buck2_events::BuckEvent; - use buck2_util::cleanup_ctx::AsyncCleanupContext; - use buck2_wrapper_common::invocation_id::TraceId; - use dupe::Dupe; - use fbinit::FacebookInit; - use futures::FutureExt; - use gazebo::variants::VariantName; - use itertools::Itertools; - use termwiz::istty::IsTty; - - use crate::build_count::BuildCountManager; - use crate::subscribers::observer::ErrorObserver; - use crate::subscribers::recorder::system_memory_stats; - use crate::subscribers::subscriber::EventSubscriber; - - pub struct InvocationRecorder<'a> { +impl<'a> InvocationRecorder<'a> { + pub fn new( fb: FacebookInit, + async_cleanup_context: AsyncCleanupContext<'a>, write_to_path: Option, command_name: &'static str, - cli_args: Vec, + sanitized_argv: Vec, + trace_id: TraceId, isolation_dir: String, - start_time: Instant, - async_cleanup_context: AsyncCleanupContext<'a>, build_count_manager: BuildCountManager, - trace_id: TraceId, - command_end: Option, - command_duration: Option, - re_session_id: Option, - re_experiment_name: Option, - critical_path_duration: Option, - tags: Vec, - run_local_count: u64, - run_remote_count: u64, - run_action_cache_count: u64, - run_remote_dep_file_cache_count: u64, - run_skipped_count: u64, - run_fallback_count: u64, - local_actions_executed_via_worker: u64, - first_snapshot: Option, - last_snapshot: Option, - min_build_count_since_rebase: u64, - cache_upload_count: u64, - cache_upload_attempt_count: u64, - parsed_target_patterns: Option, filesystem: String, - watchman_version: Option, - eden_version: Option, - test_info: Option, - eligible_for_full_hybrid: bool, - max_event_client_delay: Option, - max_malloc_bytes_active: Option, - max_malloc_bytes_allocated: Option, - run_command_failure_count: u64, - event_count: u64, - time_to_first_action_execution: Option, - materialization_output_size: u64, - initial_materializer_entries_from_sqlite: Option, - time_to_command_start: Option, - time_to_command_critical_section: Option, - time_to_first_analysis: Option, - time_to_load_first_build_file: Option, - time_to_first_command_execution_start: Option, - time_to_first_test_discovery: Option, - system_total_memory_bytes: Option, - file_watcher_stats: Option, - time_to_last_action_execution_end: Option, - initial_sink_success_count: Option, - initial_sink_failure_count: Option, - initial_sink_dropped_count: Option, - sink_max_buffer_depth: u64, - soft_error_categories: HashSet, - concurrent_command_blocking_duration: Option, - metadata: HashMap, - analysis_count: u64, - daemon_in_memory_state_is_corrupted: bool, - daemon_materializer_state_is_corrupted: bool, - enable_restarter: bool, restarted_trace_id: Option, - has_command_result: bool, - has_end_of_stream: bool, - compressed_event_log_size_bytes: Option>, - critical_path_backend: Option, - instant_command_is_success: Option, - bxl_ensure_artifacts_duration: Option, - initial_re_upload_bytes: Option, - initial_re_download_bytes: Option, - concurrent_command_ids: HashSet, - daemon_connection_failure: bool, + log_size_counter_bytes: Option>, client_metadata: Vec, - errors: Vec, - target_rule_type_names: Vec, - } - - impl<'a> InvocationRecorder<'a> { - pub fn new( - fb: FacebookInit, - async_cleanup_context: AsyncCleanupContext<'a>, - write_to_path: Option, - command_name: &'static str, - sanitized_argv: Vec, - trace_id: TraceId, - isolation_dir: String, - build_count_manager: BuildCountManager, - filesystem: String, - restarted_trace_id: Option, - log_size_counter_bytes: Option>, - client_metadata: Vec, - ) -> Self { - Self { - fb, - write_to_path, - command_name, - cli_args: sanitized_argv, - isolation_dir, - start_time: Instant::now(), - async_cleanup_context, - build_count_manager, - trace_id, - command_end: None, - command_duration: None, - re_session_id: None, - re_experiment_name: None, - critical_path_duration: None, - tags: vec![], - run_local_count: 0, - run_remote_count: 0, - run_action_cache_count: 0, - run_remote_dep_file_cache_count: 0, - run_skipped_count: 0, - run_fallback_count: 0, - local_actions_executed_via_worker: 0, - first_snapshot: None, - last_snapshot: None, - min_build_count_since_rebase: 0, - cache_upload_count: 0, - cache_upload_attempt_count: 0, - parsed_target_patterns: None, - filesystem, - watchman_version: None, - eden_version: None, - test_info: None, - eligible_for_full_hybrid: false, - max_event_client_delay: None, - max_malloc_bytes_active: None, - max_malloc_bytes_allocated: None, - run_command_failure_count: 0, - event_count: 0, - time_to_first_action_execution: None, - materialization_output_size: 0, - initial_materializer_entries_from_sqlite: None, - time_to_command_start: None, - time_to_command_critical_section: None, - time_to_first_analysis: None, - time_to_load_first_build_file: None, - time_to_first_command_execution_start: None, - time_to_first_test_discovery: None, - system_total_memory_bytes: Some(system_memory_stats()), - file_watcher_stats: None, - time_to_last_action_execution_end: None, - initial_sink_success_count: None, - initial_sink_failure_count: None, - initial_sink_dropped_count: None, - sink_max_buffer_depth: 0, - soft_error_categories: HashSet::new(), - concurrent_command_blocking_duration: None, - metadata: buck2_events::metadata::collect(), - analysis_count: 0, - daemon_in_memory_state_is_corrupted: false, - daemon_materializer_state_is_corrupted: false, - enable_restarter: false, - restarted_trace_id, - has_command_result: false, - has_end_of_stream: false, - compressed_event_log_size_bytes: log_size_counter_bytes, - critical_path_backend: None, - instant_command_is_success: None, - bxl_ensure_artifacts_duration: None, - initial_re_upload_bytes: None, - initial_re_download_bytes: None, - concurrent_command_ids: HashSet::new(), - daemon_connection_failure: false, - client_metadata, - errors: Vec::new(), - target_rule_type_names: Vec::new(), - } + ) -> Self { + Self { + fb, + write_to_path, + command_name, + cli_args: sanitized_argv, + isolation_dir, + start_time: Instant::now(), + async_cleanup_context, + build_count_manager, + trace_id, + command_end: None, + command_duration: None, + re_session_id: None, + re_experiment_name: None, + critical_path_duration: None, + tags: vec![], + run_local_count: 0, + run_remote_count: 0, + run_action_cache_count: 0, + run_remote_dep_file_cache_count: 0, + run_skipped_count: 0, + run_fallback_count: 0, + local_actions_executed_via_worker: 0, + first_snapshot: None, + last_snapshot: None, + min_build_count_since_rebase: 0, + cache_upload_count: 0, + cache_upload_attempt_count: 0, + parsed_target_patterns: None, + filesystem, + watchman_version: None, + eden_version: None, + test_info: None, + eligible_for_full_hybrid: false, + max_event_client_delay: None, + max_malloc_bytes_active: None, + max_malloc_bytes_allocated: None, + run_command_failure_count: 0, + event_count: 0, + time_to_first_action_execution: None, + materialization_output_size: 0, + initial_materializer_entries_from_sqlite: None, + time_to_command_start: None, + time_to_command_critical_section: None, + time_to_first_analysis: None, + time_to_load_first_build_file: None, + time_to_first_command_execution_start: None, + time_to_first_test_discovery: None, + system_total_memory_bytes: Some(system_memory_stats()), + file_watcher_stats: None, + time_to_last_action_execution_end: None, + initial_sink_success_count: None, + initial_sink_failure_count: None, + initial_sink_dropped_count: None, + sink_max_buffer_depth: 0, + soft_error_categories: HashSet::new(), + concurrent_command_blocking_duration: None, + metadata: buck2_events::metadata::collect(), + analysis_count: 0, + daemon_in_memory_state_is_corrupted: false, + daemon_materializer_state_is_corrupted: false, + enable_restarter: false, + restarted_trace_id, + has_command_result: false, + has_end_of_stream: false, + compressed_event_log_size_bytes: log_size_counter_bytes, + critical_path_backend: None, + instant_command_is_success: None, + bxl_ensure_artifacts_duration: None, + initial_re_upload_bytes: None, + initial_re_download_bytes: None, + concurrent_command_ids: HashSet::new(), + daemon_connection_failure: false, + daemon_was_started: None, + client_metadata, + errors: Vec::new(), + server_stderr: String::new(), + target_rule_type_names: Vec::new(), } + } - pub fn instant_command_outcome(&mut self, is_success: bool) { - self.instant_command_is_success = Some(is_success); - } + pub fn instant_command_outcome(&mut self, is_success: bool) { + self.instant_command_is_success = Some(is_success); + } - async fn build_count( - &mut self, - is_success: bool, - command_name: &str, - ) -> anyhow::Result { - if let Some(stats) = &self.file_watcher_stats { - if let Some(merge_base) = &stats.branched_from_revision { - match &self.parsed_target_patterns { - None => { - if is_success { - return Err(anyhow::anyhow!( - "successful {} commands should have resolved target patterns", - command_name - )); - } - // fallthrough to 0 below + async fn build_count(&mut self, is_success: bool, command_name: &str) -> anyhow::Result { + if let Some(stats) = &self.file_watcher_stats { + if let Some(merge_base) = &stats.branched_from_revision { + match &self.parsed_target_patterns { + None => { + if is_success { + return Err(anyhow::anyhow!( + "successful {} commands should have resolved target patterns", + command_name + )); } - Some(v) => { - return self - .build_count_manager - .min_build_count(merge_base, v, is_success) - .await - .context("Error recording build count"); - } - }; - } + // fallthrough to 0 below + } + Some(v) => { + return self + .build_count_manager + .min_build_count(merge_base, v, is_success) + .await + .context("Error recording build count"); + } + }; } - - Ok(0) } - fn send_it(&mut self) -> Option + 'static + Send> { - let mut sink_success_count = None; - let mut sink_failure_count = None; - let mut sink_dropped_count = None; - let mut re_upload_bytes = None; - let mut re_download_bytes = None; - if let Some(snapshot) = &self.last_snapshot { - sink_success_count = calculate_diff_if_some( - &snapshot.sink_successes, - &self.initial_sink_success_count, - ); - sink_failure_count = calculate_diff_if_some( - &snapshot.sink_failures, - &self.initial_sink_failure_count, - ); - sink_dropped_count = calculate_diff_if_some( - &snapshot.sink_dropped, - &self.initial_sink_dropped_count, - ); - re_upload_bytes = calculate_diff_if_some( - &Some(snapshot.re_upload_bytes), - &self.initial_re_upload_bytes, - ); - re_download_bytes = calculate_diff_if_some( - &Some(snapshot.re_download_bytes), - &self.initial_re_download_bytes, - ); - } + Ok(0) + } - let mut metadata = Self::default_metadata(); - metadata.strings.extend(std::mem::take(&mut self.metadata)); - - let record = buck2_data::InvocationRecord { - command_name: Some(self.command_name.to_owned()), - command_end: self.command_end.take(), - command_duration: self.command_duration.take(), - client_walltime: self.start_time.elapsed().try_into().ok(), - re_session_id: self.re_session_id.take().unwrap_or_default(), - re_experiment_name: self.re_experiment_name.take().unwrap_or_default(), - cli_args: self.cli_args.clone(), - critical_path_duration: self.critical_path_duration.and_then(|x| x.try_into().ok()), - metadata: Some(metadata), - tags: self.tags.drain(..).collect(), - run_local_count: self.run_local_count, - run_remote_count: self.run_remote_count, - run_action_cache_count: self.run_action_cache_count, - run_remote_dep_file_cache_count: self.run_remote_dep_file_cache_count, - cache_hit_rate: total_cache_hit_rate( - self.run_local_count, - self.run_remote_count, - self.run_action_cache_count, - self.run_remote_dep_file_cache_count, - ) as f32, - run_skipped_count: self.run_skipped_count, - run_fallback_count: Some(self.run_fallback_count), - local_actions_executed_via_worker: Some(self.local_actions_executed_via_worker), - first_snapshot: self.first_snapshot.take(), - last_snapshot: self.last_snapshot.take(), - min_build_count_since_rebase: self.min_build_count_since_rebase, - cache_upload_count: self.cache_upload_count, - cache_upload_attempt_count: self.cache_upload_attempt_count, - parsed_target_patterns: self.parsed_target_patterns.take(), - filesystem: std::mem::take(&mut self.filesystem), - watchman_version: self.watchman_version.take(), - eden_version: self.eden_version.take(), - test_info: self.test_info.take(), - eligible_for_full_hybrid: Some(self.eligible_for_full_hybrid), - max_event_client_delay_ms: self - .max_event_client_delay - .and_then(|d| u64::try_from(d.as_millis()).ok()), - max_malloc_bytes_active: self.max_malloc_bytes_active.take(), - max_malloc_bytes_allocated: self.max_malloc_bytes_allocated.take(), - run_command_failure_count: Some(self.run_command_failure_count), - event_count: Some(self.event_count), - time_to_first_action_execution_ms: self - .time_to_first_action_execution - .and_then(|d| u64::try_from(d.as_millis()).ok()), - materialization_output_size: Some(self.materialization_output_size), - initial_materializer_entries_from_sqlite: self - .initial_materializer_entries_from_sqlite, - time_to_command_start_ms: self - .time_to_command_start - .and_then(|d| u64::try_from(d.as_millis()).ok()), - time_to_command_critical_section_ms: self - .time_to_command_critical_section - .and_then(|d| u64::try_from(d.as_millis()).ok()), - time_to_first_analysis_ms: self - .time_to_first_analysis - .and_then(|d| u64::try_from(d.as_millis()).ok()), - time_to_load_first_build_file_ms: self - .time_to_load_first_build_file - .and_then(|d| u64::try_from(d.as_millis()).ok()), - time_to_first_command_execution_start_ms: self - .time_to_first_command_execution_start - .and_then(|d| u64::try_from(d.as_millis()).ok()), - time_to_first_test_discovery_ms: self - .time_to_first_test_discovery - .and_then(|d| u64::try_from(d.as_millis()).ok()), - system_total_memory_bytes: self.system_total_memory_bytes, - file_watcher_stats: self.file_watcher_stats.take(), - time_to_last_action_execution_end_ms: self - .time_to_last_action_execution_end - .and_then(|d| u64::try_from(d.as_millis()).ok()), - isolation_dir: Some(self.isolation_dir.clone()), - sink_success_count, - sink_failure_count, - sink_dropped_count, - sink_max_buffer_depth: Some(self.sink_max_buffer_depth), - soft_error_categories: std::mem::take(&mut self.soft_error_categories) - .into_iter() - .collect(), - concurrent_command_blocking_duration: self - .concurrent_command_blocking_duration - .and_then(|x| x.try_into().ok()), - analysis_count: Some(self.analysis_count), - restarted_trace_id: self.restarted_trace_id.as_ref().map(|t| t.to_string()), - has_command_result: Some(self.has_command_result), - has_end_of_stream: Some(self.has_end_of_stream), - // At this point we expect the event log writer to have finished - compressed_event_log_size_bytes: Some( - self.compressed_event_log_size_bytes - .as_ref() - .map(|x| x.load(Ordering::Relaxed)) - .unwrap_or_default(), - ), - critical_path_backend: self.critical_path_backend.take(), - instant_command_is_success: self.instant_command_is_success.take(), - bxl_ensure_artifacts_duration: self.bxl_ensure_artifacts_duration.take(), - re_upload_bytes, - re_download_bytes, - concurrent_command_ids: std::mem::take(&mut self.concurrent_command_ids) - .into_iter() - .collect(), - daemon_connection_failure: Some(self.daemon_connection_failure), - client_metadata: std::mem::take(&mut self.client_metadata), - errors: std::mem::take(&mut self.errors), - target_rule_type_names: std::mem::take(&mut self.target_rule_type_names), - }; - - let event = BuckEvent::new( - SystemTime::now(), - self.trace_id.dupe(), - None, - None, - buck2_data::RecordEvent { - data: Some((Box::new(record)).into()), - } - .into(), - ); + fn maybe_add_server_stderr_to_errors(&mut self) { + for error in &mut self.errors { + if !error.want_stderr { + continue; + } - if let Some(path) = &self.write_to_path { - let res = (|| { - let out = fs_util::create_file(path).context("Error opening")?; - let mut out = std::io::BufWriter::new(out); - serde_json::to_writer(&mut out, event.event()).context("Error writing")?; - out.flush().context("Error flushing")?; - anyhow::Ok(()) - })(); - - if let Err(e) = &res { - tracing::warn!( - "Failed to write InvocationRecord to `{}`: {:#}", - path.as_path().display(), - e - ); - } + if error.processed.message.is_empty() { + error.processed.message = + "Error is empty? But it is too late to do anything about it\n".to_owned(); + } else if !error.processed.message.ends_with('\n') { + error.processed.message.push('\n'); } - if let Ok(Some(scribe_sink)) = - new_thrift_scribe_sink_if_enabled(self.fb, 1, Duration::from_millis(500), 5, None) - { - tracing::info!("Recording invocation to Scribe: {:?}", &event); - Some(async move { - scribe_sink.send_now(event).await; - }) + error.processed.message.push('\n'); + + if self.server_stderr.is_empty() { + error.processed.message.push_str("buckd stderr is empty\n"); } else { - tracing::info!("Invocation record is not sent to Scribe: {:?}", &event); - None + error.processed.message.push_str("buckd stderr:\n"); + // Scribe sink truncates messages, but here we can do it better: + // - truncate even if total message is not large enough + // - truncate stderr, but keep the error message + let server_stderr = truncate_stderr(&self.server_stderr); + error.processed.message.push_str(server_stderr); } - } - // Collects client-side state and data, suitable for telemetry. - // NOTE: If data is visible from the daemon, put it in cli::metadata::collect() - fn default_metadata() -> buck2_data::TypedMetadata { - let mut ints = HashMap::new(); - ints.insert("is_tty".to_owned(), std::io::stderr().is_tty() as i64); - buck2_data::TypedMetadata { - ints, - strings: HashMap::new(), - } + let stderr_tag = classify_server_stderr(&self.server_stderr); + // Note: side effect, `best_error_tag` must be called after this function. + error.best_tag = best_tag(error.best_tag.into_iter().chain(iter::once(stderr_tag))); + error + .processed + .tags + .push(stderr_tag.as_str_name().to_owned()); } + } - fn handle_command_start( - &mut self, - command: &buck2_data::CommandStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.metadata.extend(command.metadata.clone()); - self.time_to_command_start = Some(self.start_time.elapsed()); - Ok(()) + fn best_error_tag(&self) -> Option<&'static str> { + if self.errors.is_empty() { + None + } else { + Some( + best_tag(self.errors.iter().filter_map(|e| e.best_tag)).map_or( + // If we don't have tags on the errors, + // we still want to add a tag to Scuba column. + ERROR_TAG_UNCLASSIFIED, + |t| t.as_str_name(), + ), + ) } + } - async fn handle_command_end( - &mut self, - command: &buck2_data::CommandEnd, - event: &BuckEvent, - ) -> anyhow::Result<()> { - let mut command = command.clone(); - self.errors.extend( - std::mem::take(&mut command.errors) - .into_iter() - .map(process_error_report), + fn send_it(&mut self) -> Option + 'static + Send> { + self.maybe_add_server_stderr_to_errors(); + + // `None` if no errors, `Some("UNCLASSIFIED")` if no tags. + let best_error_tag = self.best_error_tag(); + + let mut sink_success_count = None; + let mut sink_failure_count = None; + let mut sink_dropped_count = None; + let mut re_upload_bytes = None; + let mut re_download_bytes = None; + if let Some(snapshot) = &self.last_snapshot { + sink_success_count = + calculate_diff_if_some(&snapshot.sink_successes, &self.initial_sink_success_count); + sink_failure_count = + calculate_diff_if_some(&snapshot.sink_failures, &self.initial_sink_failure_count); + sink_dropped_count = + calculate_diff_if_some(&snapshot.sink_dropped, &self.initial_sink_dropped_count); + re_upload_bytes = calculate_diff_if_some( + &Some(snapshot.re_upload_bytes), + &self.initial_re_upload_bytes, + ); + re_download_bytes = calculate_diff_if_some( + &Some(snapshot.re_download_bytes), + &self.initial_re_download_bytes, ); - - // Awkwardly unpacks the SpanEnd event so we can read its duration. - let command_end = match event.data() { - buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), - _ => { - return Err(anyhow::anyhow!( - "handle_command_end was passed a CommandEnd not contained in a SpanEndEvent" - )); - } - }; - self.command_duration = command_end.duration; - let command_data = command.data.as_ref().context("Missing command data")?; - self.min_build_count_since_rebase = match command_data { - buck2_data::command_end::Data::Build(..) - | buck2_data::command_end::Data::Test(..) - | buck2_data::command_end::Data::Install(..) => { - self.build_count(command.is_success, command_data.variant_name()) - .await? - } - // other events don't count builds - _ => 0, - }; - self.command_end = Some(command); - Ok(()) } - fn handle_command_critical_start( - &mut self, - command: &buck2_data::CommandCriticalStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.metadata.extend(command.metadata.clone()); - self.time_to_command_critical_section = Some(self.start_time.elapsed()); - Ok(()) + + let mut metadata = Self::default_metadata(); + metadata.strings.extend(std::mem::take(&mut self.metadata)); + + let record = buck2_data::InvocationRecord { + command_name: Some(self.command_name.to_owned()), + command_end: self.command_end.take(), + command_duration: self.command_duration.take(), + client_walltime: self.start_time.elapsed().try_into().ok(), + re_session_id: self.re_session_id.take().unwrap_or_default(), + re_experiment_name: self.re_experiment_name.take().unwrap_or_default(), + cli_args: self.cli_args.clone(), + critical_path_duration: self.critical_path_duration.and_then(|x| x.try_into().ok()), + metadata: Some(metadata), + tags: self.tags.drain(..).collect(), + run_local_count: self.run_local_count, + run_remote_count: self.run_remote_count, + run_action_cache_count: self.run_action_cache_count, + run_remote_dep_file_cache_count: self.run_remote_dep_file_cache_count, + cache_hit_rate: total_cache_hit_rate( + self.run_local_count, + self.run_remote_count, + self.run_action_cache_count, + self.run_remote_dep_file_cache_count, + ) as f32, + run_skipped_count: self.run_skipped_count, + run_fallback_count: Some(self.run_fallback_count), + local_actions_executed_via_worker: Some(self.local_actions_executed_via_worker), + first_snapshot: self.first_snapshot.take(), + last_snapshot: self.last_snapshot.take(), + min_build_count_since_rebase: self.min_build_count_since_rebase, + cache_upload_count: self.cache_upload_count, + cache_upload_attempt_count: self.cache_upload_attempt_count, + parsed_target_patterns: self.parsed_target_patterns.take(), + filesystem: std::mem::take(&mut self.filesystem), + watchman_version: self.watchman_version.take(), + eden_version: self.eden_version.take(), + test_info: self.test_info.take(), + eligible_for_full_hybrid: Some(self.eligible_for_full_hybrid), + max_event_client_delay_ms: self + .max_event_client_delay + .and_then(|d| u64::try_from(d.as_millis()).ok()), + max_malloc_bytes_active: self.max_malloc_bytes_active.take(), + max_malloc_bytes_allocated: self.max_malloc_bytes_allocated.take(), + run_command_failure_count: Some(self.run_command_failure_count), + event_count: Some(self.event_count), + time_to_first_action_execution_ms: self + .time_to_first_action_execution + .and_then(|d| u64::try_from(d.as_millis()).ok()), + materialization_output_size: Some(self.materialization_output_size), + initial_materializer_entries_from_sqlite: self.initial_materializer_entries_from_sqlite, + time_to_command_start_ms: self + .time_to_command_start + .and_then(|d| u64::try_from(d.as_millis()).ok()), + time_to_command_critical_section_ms: self + .time_to_command_critical_section + .and_then(|d| u64::try_from(d.as_millis()).ok()), + time_to_first_analysis_ms: self + .time_to_first_analysis + .and_then(|d| u64::try_from(d.as_millis()).ok()), + time_to_load_first_build_file_ms: self + .time_to_load_first_build_file + .and_then(|d| u64::try_from(d.as_millis()).ok()), + time_to_first_command_execution_start_ms: self + .time_to_first_command_execution_start + .and_then(|d| u64::try_from(d.as_millis()).ok()), + time_to_first_test_discovery_ms: self + .time_to_first_test_discovery + .and_then(|d| u64::try_from(d.as_millis()).ok()), + system_total_memory_bytes: self.system_total_memory_bytes, + file_watcher_stats: self.file_watcher_stats.take(), + time_to_last_action_execution_end_ms: self + .time_to_last_action_execution_end + .and_then(|d| u64::try_from(d.as_millis()).ok()), + isolation_dir: Some(self.isolation_dir.clone()), + sink_success_count, + sink_failure_count, + sink_dropped_count, + sink_max_buffer_depth: Some(self.sink_max_buffer_depth), + soft_error_categories: std::mem::take(&mut self.soft_error_categories) + .into_iter() + .collect(), + concurrent_command_blocking_duration: self + .concurrent_command_blocking_duration + .and_then(|x| x.try_into().ok()), + analysis_count: Some(self.analysis_count), + restarted_trace_id: self.restarted_trace_id.as_ref().map(|t| t.to_string()), + has_command_result: Some(self.has_command_result), + has_end_of_stream: Some(self.has_end_of_stream), + // At this point we expect the event log writer to have finished + compressed_event_log_size_bytes: Some( + self.compressed_event_log_size_bytes + .as_ref() + .map(|x| x.load(Ordering::Relaxed)) + .unwrap_or_default(), + ), + critical_path_backend: self.critical_path_backend.take(), + instant_command_is_success: self.instant_command_is_success.take(), + bxl_ensure_artifacts_duration: self.bxl_ensure_artifacts_duration.take(), + re_upload_bytes, + re_download_bytes, + concurrent_command_ids: std::mem::take(&mut self.concurrent_command_ids) + .into_iter() + .collect(), + daemon_connection_failure: Some(self.daemon_connection_failure), + daemon_was_started: self.daemon_was_started.map(|t| t as i32), + client_metadata: std::mem::take(&mut self.client_metadata), + errors: std::mem::take(&mut self.errors).into_map(|e| e.processed), + best_error_tag: best_error_tag.map(|t| t.to_owned()), + target_rule_type_names: std::mem::take(&mut self.target_rule_type_names), + }; + + let event = BuckEvent::new( + SystemTime::now(), + self.trace_id.dupe(), + None, + None, + buck2_data::RecordEvent { + data: Some((Box::new(record)).into()), + } + .into(), + ); + + if let Some(path) = &self.write_to_path { + let res = (|| { + let out = fs_util::create_file(path).context("Error opening")?; + let mut out = std::io::BufWriter::new(out); + serde_json::to_writer(&mut out, event.event()).context("Error writing")?; + out.flush().context("Error flushing")?; + anyhow::Ok(()) + })(); + + if let Err(e) = &res { + tracing::warn!( + "Failed to write InvocationRecord to `{}`: {:#}", + path.as_path().display(), + e + ); + } } - fn handle_command_critical_end( - &mut self, - command: &buck2_data::CommandCriticalEnd, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.metadata.extend(command.metadata.clone()); - Ok(()) + + if let Ok(Some(scribe_sink)) = + new_thrift_scribe_sink_if_enabled(self.fb, 1, Duration::from_millis(500), 5, None) + { + tracing::info!("Recording invocation to Scribe: {:?}", &event); + Some(async move { + scribe_sink.send_now(event).await; + }) + } else { + tracing::info!("Invocation record is not sent to Scribe: {:?}", &event); + None } + } - fn handle_action_execution_start( - &mut self, - _action: &buck2_data::ActionExecutionStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - if self.time_to_first_action_execution.is_none() { - self.time_to_first_action_execution = Some(self.start_time.elapsed()); - } - Ok(()) + // Collects client-side state and data, suitable for telemetry. + // NOTE: If data is visible from the daemon, put it in cli::metadata::collect() + fn default_metadata() -> buck2_data::TypedMetadata { + let mut ints = HashMap::new(); + ints.insert("is_tty".to_owned(), std::io::stderr().is_tty() as i64); + buck2_data::TypedMetadata { + ints, + strings: HashMap::new(), } - fn handle_action_execution_end( - &mut self, - action: &buck2_data::ActionExecutionEnd, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - if action.kind == buck2_data::ActionKind::Run as i32 { - if action_stats::was_fallback_action(action) { - self.run_fallback_count += 1; - } + } - match last_command_execution_kind::get_last_command_execution_kind(action) { - LastCommandExecutionKind::Local => { - self.run_local_count += 1; - } - LastCommandExecutionKind::LocalWorker => { - self.run_local_count += 1; - self.local_actions_executed_via_worker += 1; - } - LastCommandExecutionKind::Cached => { - self.run_action_cache_count += 1; - } - LastCommandExecutionKind::RemoteDepFileCached => { - self.run_remote_dep_file_cache_count += 1; - } - LastCommandExecutionKind::Remote => { - self.run_remote_count += 1; - } - LastCommandExecutionKind::NoCommand => { - self.run_skipped_count += 1; - } + fn handle_command_start( + &mut self, + command: &buck2_data::CommandStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.metadata.extend(command.metadata.clone()); + self.time_to_command_start = Some(self.start_time.elapsed()); + Ok(()) + } + + async fn handle_command_end( + &mut self, + command: &buck2_data::CommandEnd, + event: &BuckEvent, + ) -> anyhow::Result<()> { + let mut command = command.clone(); + self.errors + .extend(std::mem::take(&mut command.errors).into_iter().map(|e| { + let best_tag = best_tag(e.tags.iter().filter_map(|t| { + // This should never be `None`, but with weak prost types, + // it is safer to just ignore incorrect integers. + ErrorTag::from_i32(*t) + })); + ErrorIntermediate { + processed: process_error_report(e), + want_stderr: false, + best_tag, } + })); + + // Awkwardly unpacks the SpanEnd event so we can read its duration. + let command_end = match event.data() { + buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), + _ => { + return Err(anyhow::anyhow!( + "handle_command_end was passed a CommandEnd not contained in a SpanEndEvent" + )); } - - if action.eligible_for_full_hybrid.unwrap_or_default() { - self.eligible_for_full_hybrid = true; + }; + self.command_duration = command_end.duration; + let command_data = command.data.as_ref().context("Missing command data")?; + self.min_build_count_since_rebase = match command_data { + buck2_data::command_end::Data::Build(..) + | buck2_data::command_end::Data::Test(..) + | buck2_data::command_end::Data::Install(..) => { + self.build_count(command.is_success, command_data.variant_name()) + .await? } + // other events don't count builds + _ => 0, + }; + self.command_end = Some(command); + Ok(()) + } + fn handle_command_critical_start( + &mut self, + command: &buck2_data::CommandCriticalStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.metadata.extend(command.metadata.clone()); + self.time_to_command_critical_section = Some(self.start_time.elapsed()); + Ok(()) + } + fn handle_command_critical_end( + &mut self, + command: &buck2_data::CommandCriticalEnd, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.metadata.extend(command.metadata.clone()); + Ok(()) + } - if action.commands.iter().any(|c| { - matches!( - c.status, - Some(buck2_data::command_execution::Status::Failure(..)) - ) - }) { - self.run_command_failure_count += 1; + fn handle_action_execution_start( + &mut self, + _action: &buck2_data::ActionExecutionStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + if self.time_to_first_action_execution.is_none() { + self.time_to_first_action_execution = Some(self.start_time.elapsed()); + } + Ok(()) + } + fn handle_action_execution_end( + &mut self, + action: &buck2_data::ActionExecutionEnd, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + if action.kind == buck2_data::ActionKind::Run as i32 { + if action_stats::was_fallback_action(action) { + self.run_fallback_count += 1; } - self.time_to_last_action_execution_end = Some(self.start_time.elapsed()); - - Ok(()) + match last_command_execution_kind::get_last_command_execution_kind(action) { + LastCommandExecutionKind::Local => { + self.run_local_count += 1; + } + LastCommandExecutionKind::LocalWorker => { + self.run_local_count += 1; + self.local_actions_executed_via_worker += 1; + } + LastCommandExecutionKind::Cached => { + self.run_action_cache_count += 1; + } + LastCommandExecutionKind::RemoteDepFileCached => { + self.run_remote_dep_file_cache_count += 1; + } + LastCommandExecutionKind::Remote => { + self.run_remote_count += 1; + } + LastCommandExecutionKind::NoCommand => { + self.run_skipped_count += 1; + } + } } - fn handle_analysis_start( - &mut self, - _analysis: &buck2_data::AnalysisStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.time_to_first_analysis - .get_or_insert_with(|| self.start_time.elapsed()); - Ok(()) + if action.eligible_for_full_hybrid.unwrap_or_default() { + self.eligible_for_full_hybrid = true; } - fn handle_load_start( - &mut self, - _eval: &buck2_data::LoadBuildFileStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.time_to_load_first_build_file - .get_or_insert_with(|| self.start_time.elapsed()); - Ok(()) + if action.commands.iter().any(|c| { + matches!( + c.status, + Some(buck2_data::command_execution::Status::Failure(..)) + ) + }) { + self.run_command_failure_count += 1; } - fn handle_executor_stage_start( - &mut self, - executor_stage: &buck2_data::ExecutorStageStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - match &executor_stage.stage { - Some(buck2_data::executor_stage_start::Stage::Re(re_stage)) => { - match &re_stage.stage { - Some(buck2_data::re_stage::Stage::Execute(_)) => { - self.time_to_first_command_execution_start - .get_or_insert_with(|| self.start_time.elapsed()); - } - _ => {} - } + self.time_to_last_action_execution_end = Some(self.start_time.elapsed()); + + Ok(()) + } + + fn handle_analysis_start( + &mut self, + _analysis: &buck2_data::AnalysisStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.time_to_first_analysis + .get_or_insert_with(|| self.start_time.elapsed()); + Ok(()) + } + + fn handle_load_start( + &mut self, + _eval: &buck2_data::LoadBuildFileStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.time_to_load_first_build_file + .get_or_insert_with(|| self.start_time.elapsed()); + Ok(()) + } + + fn handle_executor_stage_start( + &mut self, + executor_stage: &buck2_data::ExecutorStageStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + match &executor_stage.stage { + Some(buck2_data::executor_stage_start::Stage::Re(re_stage)) => match &re_stage.stage { + Some(buck2_data::re_stage::Stage::Execute(_)) => { + self.time_to_first_command_execution_start + .get_or_insert_with(|| self.start_time.elapsed()); } - Some(buck2_data::executor_stage_start::Stage::Local(local_stage)) => { - match &local_stage.stage { - Some(buck2_data::local_stage::Stage::Execute(_)) => { - self.time_to_first_command_execution_start - .get_or_insert_with(|| self.start_time.elapsed()); - } - _ => {} + _ => {} + }, + Some(buck2_data::executor_stage_start::Stage::Local(local_stage)) => { + match &local_stage.stage { + Some(buck2_data::local_stage::Stage::Execute(_)) => { + self.time_to_first_command_execution_start + .get_or_insert_with(|| self.start_time.elapsed()); } + _ => {} } - _ => {} } - Ok(()) + _ => {} } + Ok(()) + } - fn handle_cache_upload_end( - &mut self, - cache_upload: &buck2_data::CacheUploadEnd, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - if cache_upload.success { - self.cache_upload_count += 1; - } - self.cache_upload_attempt_count += 1; - Ok(()) + fn handle_cache_upload_end( + &mut self, + cache_upload: &buck2_data::CacheUploadEnd, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + if cache_upload.success { + self.cache_upload_count += 1; } + self.cache_upload_attempt_count += 1; + Ok(()) + } - fn handle_re_session_created( - &mut self, - session: &buck2_data::RemoteExecutionSessionCreated, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.re_session_id = Some(session.session_id.clone()); - self.re_experiment_name = Some(session.experiment_name.clone()); - Ok(()) - } + fn handle_re_session_created( + &mut self, + session: &buck2_data::RemoteExecutionSessionCreated, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.re_session_id = Some(session.session_id.clone()); + self.re_experiment_name = Some(session.experiment_name.clone()); + Ok(()) + } - fn handle_materialization_end( - &mut self, - materialization: &buck2_data::MaterializationEnd, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.materialization_output_size += materialization.total_bytes; - Ok(()) - } + fn handle_materialization_end( + &mut self, + materialization: &buck2_data::MaterializationEnd, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.materialization_output_size += materialization.total_bytes; + Ok(()) + } - fn handle_materializer_state_info( - &mut self, - materializer_state_info: &buck2_data::MaterializerStateInfo, - ) -> anyhow::Result<()> { - self.initial_materializer_entries_from_sqlite = - Some(materializer_state_info.num_entries_from_sqlite); - Ok(()) - } + fn handle_materializer_state_info( + &mut self, + materializer_state_info: &buck2_data::MaterializerStateInfo, + ) -> anyhow::Result<()> { + self.initial_materializer_entries_from_sqlite = + Some(materializer_state_info.num_entries_from_sqlite); + Ok(()) + } - fn handle_bxl_ensure_artifacts_end( - &mut self, - _bxl_ensure_artifacts_end: &buck2_data::BxlEnsureArtifactsEnd, - event: &BuckEvent, - ) -> anyhow::Result<()> { - let bxl_ensure_artifacts_end = match event.data() { - buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), - _ => { - return Err(anyhow::anyhow!( - "handle_bxl_ensure_artifacts_end was passed a BxlEnsureArtifacts not contained in a SpanEndEvent" - )); - } - }; + fn handle_bxl_ensure_artifacts_end( + &mut self, + _bxl_ensure_artifacts_end: &buck2_data::BxlEnsureArtifactsEnd, + event: &BuckEvent, + ) -> anyhow::Result<()> { + let bxl_ensure_artifacts_end = match event.data() { + buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), + _ => { + return Err(anyhow::anyhow!( + "handle_bxl_ensure_artifacts_end was passed a BxlEnsureArtifacts not contained in a SpanEndEvent" + )); + } + }; - self.bxl_ensure_artifacts_duration = bxl_ensure_artifacts_end.duration; - Ok(()) - } + self.bxl_ensure_artifacts_duration = bxl_ensure_artifacts_end.duration; + Ok(()) + } - fn handle_test_discovery( - &mut self, - test_info: &buck2_data::TestDiscovery, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - match &test_info.data { - Some(buck2_data::test_discovery::Data::Session(session_info)) => { - self.test_info = Some(session_info.info.clone()); - } - Some(buck2_data::test_discovery::Data::Tests(..)) | None => {} + fn handle_test_discovery( + &mut self, + test_info: &buck2_data::TestDiscovery, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + match &test_info.data { + Some(buck2_data::test_discovery::Data::Session(session_info)) => { + self.test_info = Some(session_info.info.clone()); } - - Ok(()) + Some(buck2_data::test_discovery::Data::Tests(..)) | None => {} } - fn handle_test_discovery_start( - &mut self, - _test_discovery: &buck2_data::TestDiscoveryStart, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.time_to_first_test_discovery - .get_or_insert_with(|| self.start_time.elapsed()); - Ok(()) - } + Ok(()) + } - fn handle_build_graph_info( - &mut self, - info: &buck2_data::BuildGraphExecutionInfo, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - let mut duration = Duration::default(); + fn handle_test_discovery_start( + &mut self, + _test_discovery: &buck2_data::TestDiscoveryStart, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.time_to_first_test_discovery + .get_or_insert_with(|| self.start_time.elapsed()); + Ok(()) + } - for node in &info.critical_path { - if let Some(d) = &node.duration { - duration += d.try_into_duration()?; - } - } + fn handle_build_graph_info( + &mut self, + info: &buck2_data::BuildGraphExecutionInfo, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + let mut duration = Duration::default(); - for node in &info.critical_path2 { - if let Some(d) = &node.duration { - duration += d.try_into_duration()?; - } + for node in &info.critical_path { + if let Some(d) = &node.duration { + duration += d.try_into_duration()?; } - - self.critical_path_duration = Some(duration); - self.critical_path_backend = info.backend_name.clone(); - Ok(()) } - fn handle_io_provider_info( - &mut self, - io_provider_info: &buck2_data::IoProviderInfo, - ) -> anyhow::Result<()> { - self.eden_version = io_provider_info.eden_version.to_owned(); - Ok(()) + for node in &info.critical_path2 { + if let Some(d) = &node.duration { + duration += d.try_into_duration()?; + } } - fn handle_tag(&mut self, tag: &buck2_data::TagEvent) -> anyhow::Result<()> { - self.tags.extend(tag.tags.iter().cloned()); - Ok(()) - } + self.critical_path_duration = Some(duration); + self.critical_path_backend = info.backend_name.clone(); + Ok(()) + } - fn handle_concurrent_commands( - &mut self, - concurrent_commands: &buck2_data::ConcurrentCommands, - ) -> anyhow::Result<()> { - concurrent_commands.trace_ids.iter().for_each(|c| { - self.concurrent_command_ids.insert(c.clone()); - }); - Ok(()) - } + fn handle_io_provider_info( + &mut self, + io_provider_info: &buck2_data::IoProviderInfo, + ) -> anyhow::Result<()> { + self.eden_version = io_provider_info.eden_version.to_owned(); + Ok(()) + } - fn handle_snapshot( - &mut self, - update: &buck2_data::Snapshot, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - self.max_malloc_bytes_active = - cmp::max(self.max_malloc_bytes_active, update.malloc_bytes_active); - self.max_malloc_bytes_allocated = cmp::max( - self.max_malloc_bytes_allocated, - update.malloc_bytes_allocated, - ); - if self.first_snapshot.is_none() { - self.first_snapshot = Some(update.clone()); - } else { - self.last_snapshot = Some(update.clone()); - } - if self.initial_sink_success_count.is_none() { - self.initial_sink_success_count = update.sink_successes; - } - if self.initial_sink_failure_count.is_none() { - self.initial_sink_failure_count = update.sink_failures; - } - if self.initial_sink_dropped_count.is_none() { - self.initial_sink_dropped_count = update.sink_dropped; - } - self.sink_max_buffer_depth = - cmp::max(self.sink_max_buffer_depth, update.sink_buffer_depth()); + fn handle_tag(&mut self, tag: &buck2_data::TagEvent) -> anyhow::Result<()> { + self.tags.extend(tag.tags.iter().cloned()); + Ok(()) + } - if self.initial_re_upload_bytes.is_none() { - self.initial_re_upload_bytes = Some(update.re_upload_bytes); - } - if self.initial_re_download_bytes.is_none() { - self.initial_re_download_bytes = Some(update.re_download_bytes); - } + fn handle_concurrent_commands( + &mut self, + concurrent_commands: &buck2_data::ConcurrentCommands, + ) -> anyhow::Result<()> { + concurrent_commands.trace_ids.iter().for_each(|c| { + self.concurrent_command_ids.insert(c.clone()); + }); + Ok(()) + } - Ok(()) + fn handle_snapshot( + &mut self, + update: &buck2_data::Snapshot, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + self.max_malloc_bytes_active = + cmp::max(self.max_malloc_bytes_active, update.malloc_bytes_active); + self.max_malloc_bytes_allocated = cmp::max( + self.max_malloc_bytes_allocated, + update.malloc_bytes_allocated, + ); + if self.first_snapshot.is_none() { + self.first_snapshot = Some(update.clone()); + } else { + self.last_snapshot = Some(update.clone()); } + if self.initial_sink_success_count.is_none() { + self.initial_sink_success_count = update.sink_successes; + } + if self.initial_sink_failure_count.is_none() { + self.initial_sink_failure_count = update.sink_failures; + } + if self.initial_sink_dropped_count.is_none() { + self.initial_sink_dropped_count = update.sink_dropped; + } + self.sink_max_buffer_depth = + cmp::max(self.sink_max_buffer_depth, update.sink_buffer_depth()); - fn handle_file_watcher_end( - &mut self, - file_watcher: &buck2_data::FileWatcherEnd, - _event: &BuckEvent, - ) -> anyhow::Result<()> { - // We might receive this event twice, so ... deal with it by merging the two. - // See: https://fb.workplace.com/groups/buck2dev/permalink/3396726613948720/ - self.file_watcher_stats = merge_file_watcher_stats( - self.file_watcher_stats.take(), - file_watcher.stats.clone(), - ); - - if let Some(stats) = &file_watcher.stats { - self.watchman_version = stats.watchman_version.to_owned(); - } - Ok(()) + if self.initial_re_upload_bytes.is_none() { + self.initial_re_upload_bytes = Some(update.re_upload_bytes); + } + if self.initial_re_download_bytes.is_none() { + self.initial_re_download_bytes = Some(update.re_download_bytes); } - fn handle_parsed_target_patterns( - &mut self, - patterns: &buck2_data::ParsedTargetPatterns, - ) -> anyhow::Result<()> { - self.parsed_target_patterns = Some(patterns.clone()); - Ok(()) + Ok(()) + } + + fn handle_file_watcher_end( + &mut self, + file_watcher: &buck2_data::FileWatcherEnd, + _event: &BuckEvent, + ) -> anyhow::Result<()> { + // We might receive this event twice, so ... deal with it by merging the two. + // See: https://fb.workplace.com/groups/buck2dev/permalink/3396726613948720/ + self.file_watcher_stats = + merge_file_watcher_stats(self.file_watcher_stats.take(), file_watcher.stats.clone()); + + if let Some(stats) = &file_watcher.stats { + self.watchman_version = stats.watchman_version.to_owned(); } + Ok(()) + } - fn handle_structured_error( - &mut self, - err: &buck2_data::StructuredError, - ) -> anyhow::Result<()> { - if let Some(soft_error_category) = err.soft_error_category.as_ref() { - self.soft_error_categories - .insert(soft_error_category.to_owned()); + fn handle_parsed_target_patterns( + &mut self, + patterns: &buck2_data::ParsedTargetPatterns, + ) -> anyhow::Result<()> { + self.parsed_target_patterns = Some(patterns.clone()); + Ok(()) + } - if err.daemon_in_memory_state_is_corrupted { - self.daemon_in_memory_state_is_corrupted = true; - } + fn handle_structured_error(&mut self, err: &buck2_data::StructuredError) -> anyhow::Result<()> { + if let Some(soft_error_category) = err.soft_error_category.as_ref() { + self.soft_error_categories + .insert(soft_error_category.to_owned()); - if err.daemon_materializer_state_is_corrupted { - self.daemon_materializer_state_is_corrupted = true; - } + if err.daemon_in_memory_state_is_corrupted { + self.daemon_in_memory_state_is_corrupted = true; } - Ok(()) + if err.daemon_materializer_state_is_corrupted { + self.daemon_materializer_state_is_corrupted = true; + } } - fn handle_dice_block_concurrent_command_end( - &mut self, - _command: &buck2_data::DiceBlockConcurrentCommandEnd, - event: &BuckEvent, - ) -> anyhow::Result<()> { - let block_concurrent_command = match event.data() { - buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), - _ => { - return Err(anyhow::anyhow!( - "handle_dice_block_concurrent_command_end was passed a DiceBlockConcurrentCommandEnd not contained in a SpanEndEvent" - )); - } - }; + Ok(()) + } - let mut duration = self - .concurrent_command_blocking_duration - .unwrap_or_default(); - if let Some(d) = &block_concurrent_command.duration { - duration += d.try_into_duration()?; + fn handle_dice_block_concurrent_command_end( + &mut self, + _command: &buck2_data::DiceBlockConcurrentCommandEnd, + event: &BuckEvent, + ) -> anyhow::Result<()> { + let block_concurrent_command = match event.data() { + buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), + _ => { + return Err(anyhow::anyhow!( + "handle_dice_block_concurrent_command_end was passed a DiceBlockConcurrentCommandEnd not contained in a SpanEndEvent" + )); } + }; - self.concurrent_command_blocking_duration = Some(duration); - - Ok(()) + let mut duration = self + .concurrent_command_blocking_duration + .unwrap_or_default(); + if let Some(d) = &block_concurrent_command.duration { + duration += d.try_into_duration()?; } - fn handle_dice_cleanup_end( - &mut self, - _command: &buck2_data::DiceCleanupEnd, - event: &BuckEvent, - ) -> anyhow::Result<()> { - let dice_cleanup_end = match event.data() { - buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), - _ => { - return Err(anyhow::anyhow!( - "handle_dice_cleanup_end was passed a DiceCleanupEnd not contained in a SpanEndEvent" - )); - } - }; + self.concurrent_command_blocking_duration = Some(duration); - let mut duration = self - .concurrent_command_blocking_duration - .unwrap_or_default(); - if let Some(d) = &dice_cleanup_end.duration { - duration += d.try_into_duration()?; - } + Ok(()) + } - self.concurrent_command_blocking_duration = Some(duration); + fn handle_dice_cleanup_end( + &mut self, + _command: &buck2_data::DiceCleanupEnd, + event: &BuckEvent, + ) -> anyhow::Result<()> { + let dice_cleanup_end = match event.data() { + buck2_data::buck_event::Data::SpanEnd(ref end) => end.clone(), + _ => { + return Err(anyhow::anyhow!( + "handle_dice_cleanup_end was passed a DiceCleanupEnd not contained in a SpanEndEvent" + )); + } + }; - Ok(()) + let mut duration = self + .concurrent_command_blocking_duration + .unwrap_or_default(); + if let Some(d) = &dice_cleanup_end.duration { + duration += d.try_into_duration()?; } - async fn handle_event(&mut self, event: &Arc) -> anyhow::Result<()> { - // TODO(nga): query now once in `EventsCtx`. - let now = SystemTime::now(); - if let Ok(delay) = now.duration_since(event.timestamp()) { - self.max_event_client_delay = Some(cmp::max( - self.max_event_client_delay.unwrap_or_default(), - delay, - )); - } - self.event_count += 1; + self.concurrent_command_blocking_duration = Some(duration); - match event.data() { - buck2_data::buck_event::Data::SpanStart(ref start) => { - match start.data.as_ref().context("Missing `start`")? { - buck2_data::span_start_event::Data::Command(command) => { - self.handle_command_start(command, event) - } - buck2_data::span_start_event::Data::CommandCritical(command) => { - self.handle_command_critical_start(command, event) - } - buck2_data::span_start_event::Data::ActionExecution(action) => { - self.handle_action_execution_start(action, event) - } - buck2_data::span_start_event::Data::Analysis(analysis) => { - self.handle_analysis_start(analysis, event) - } - buck2_data::span_start_event::Data::Load(eval) => { - self.handle_load_start(eval, event) - } - buck2_data::span_start_event::Data::ExecutorStage(stage) => { - self.handle_executor_stage_start(stage, event) - } - buck2_data::span_start_event::Data::TestDiscovery(test_discovery) => { - self.handle_test_discovery_start(test_discovery, event) - } - _ => Ok(()), + Ok(()) + } + + async fn handle_event(&mut self, event: &Arc) -> anyhow::Result<()> { + // TODO(nga): query now once in `EventsCtx`. + let now = SystemTime::now(); + if let Ok(delay) = now.duration_since(event.timestamp()) { + self.max_event_client_delay = Some(cmp::max( + self.max_event_client_delay.unwrap_or_default(), + delay, + )); + } + self.event_count += 1; + + match event.data() { + buck2_data::buck_event::Data::SpanStart(ref start) => { + match start.data.as_ref().context("Missing `start`")? { + buck2_data::span_start_event::Data::Command(command) => { + self.handle_command_start(command, event) + } + buck2_data::span_start_event::Data::CommandCritical(command) => { + self.handle_command_critical_start(command, event) + } + buck2_data::span_start_event::Data::ActionExecution(action) => { + self.handle_action_execution_start(action, event) + } + buck2_data::span_start_event::Data::Analysis(analysis) => { + self.handle_analysis_start(analysis, event) + } + buck2_data::span_start_event::Data::Load(eval) => { + self.handle_load_start(eval, event) + } + buck2_data::span_start_event::Data::ExecutorStage(stage) => { + self.handle_executor_stage_start(stage, event) + } + buck2_data::span_start_event::Data::TestDiscovery(test_discovery) => { + self.handle_test_discovery_start(test_discovery, event) } + _ => Ok(()), } - buck2_data::buck_event::Data::SpanEnd(ref end) => { - match end.data.as_ref().context("Missing `end`")? { - buck2_data::span_end_event::Data::Command(command) => { - self.handle_command_end(command, event).await - } - buck2_data::span_end_event::Data::CommandCritical(command) => { - self.handle_command_critical_end(command, event) - } - buck2_data::span_end_event::Data::ActionExecution(action) => { - self.handle_action_execution_end(action, event) - } - buck2_data::span_end_event::Data::FileWatcher(file_watcher) => { - self.handle_file_watcher_end(file_watcher, event) - } - buck2_data::span_end_event::Data::CacheUpload(cache_upload) => { - self.handle_cache_upload_end(cache_upload, event) - } - buck2_data::span_end_event::Data::Materialization(materialization) => { - self.handle_materialization_end(materialization, event) - } - buck2_data::span_end_event::Data::Analysis(..) => { - self.analysis_count += 1; - Ok(()) - } - buck2_data::span_end_event::Data::DiceBlockConcurrentCommand( - block_concurrent_command, - ) => self.handle_dice_block_concurrent_command_end( - block_concurrent_command, - event, - ), - buck2_data::span_end_event::Data::DiceCleanup(dice_cleanup_end) => { - self.handle_dice_cleanup_end(dice_cleanup_end, event) - } - buck2_data::span_end_event::Data::BxlEnsureArtifacts( - _bxl_ensure_artifacts, - ) => self.handle_bxl_ensure_artifacts_end(_bxl_ensure_artifacts, event), - _ => Ok(()), + } + buck2_data::buck_event::Data::SpanEnd(ref end) => { + match end.data.as_ref().context("Missing `end`")? { + buck2_data::span_end_event::Data::Command(command) => { + self.handle_command_end(command, event).await + } + buck2_data::span_end_event::Data::CommandCritical(command) => { + self.handle_command_critical_end(command, event) + } + buck2_data::span_end_event::Data::ActionExecution(action) => { + self.handle_action_execution_end(action, event) + } + buck2_data::span_end_event::Data::FileWatcher(file_watcher) => { + self.handle_file_watcher_end(file_watcher, event) + } + buck2_data::span_end_event::Data::CacheUpload(cache_upload) => { + self.handle_cache_upload_end(cache_upload, event) + } + buck2_data::span_end_event::Data::Materialization(materialization) => { + self.handle_materialization_end(materialization, event) + } + buck2_data::span_end_event::Data::Analysis(..) => { + self.analysis_count += 1; + Ok(()) + } + buck2_data::span_end_event::Data::DiceBlockConcurrentCommand( + block_concurrent_command, + ) => self + .handle_dice_block_concurrent_command_end(block_concurrent_command, event), + buck2_data::span_end_event::Data::DiceCleanup(dice_cleanup_end) => { + self.handle_dice_cleanup_end(dice_cleanup_end, event) } + buck2_data::span_end_event::Data::BxlEnsureArtifacts(_bxl_ensure_artifacts) => { + self.handle_bxl_ensure_artifacts_end(_bxl_ensure_artifacts, event) + } + _ => Ok(()), } - buck2_data::buck_event::Data::Instant(ref instant) => { - match instant.data.as_ref().context("Missing `data`")? { - buck2_data::instant_event::Data::ReSession(session) => { - self.handle_re_session_created(session, event) - } - buck2_data::instant_event::Data::BuildGraphInfo(info) => { - self.handle_build_graph_info(info, event) - } - buck2_data::instant_event::Data::TestDiscovery(discovery) => { - self.handle_test_discovery(discovery, event) - } - buck2_data::instant_event::Data::Snapshot(result) => { - self.handle_snapshot(result, event) - } - buck2_data::instant_event::Data::TagEvent(tag) => self.handle_tag(tag), - buck2_data::instant_event::Data::IoProviderInfo(io_provider_info) => { - self.handle_io_provider_info(io_provider_info) - } - buck2_data::instant_event::Data::TargetPatterns(tag) => { - self.handle_parsed_target_patterns(tag) - } - buck2_data::instant_event::Data::MaterializerStateInfo( - materializer_state, - ) => self.handle_materializer_state_info(materializer_state), - buck2_data::instant_event::Data::StructuredError(err) => { - self.handle_structured_error(err) - } - buck2_data::instant_event::Data::RestartConfiguration(conf) => { - self.enable_restarter = conf.enable_restarter; - Ok(()) - } - buck2_data::instant_event::Data::ConcurrentCommands( - concurrent_commands, - ) => self.handle_concurrent_commands(concurrent_commands), - _ => Ok(()), + } + buck2_data::buck_event::Data::Instant(ref instant) => { + match instant.data.as_ref().context("Missing `data`")? { + buck2_data::instant_event::Data::ReSession(session) => { + self.handle_re_session_created(session, event) + } + buck2_data::instant_event::Data::BuildGraphInfo(info) => { + self.handle_build_graph_info(info, event) + } + buck2_data::instant_event::Data::TestDiscovery(discovery) => { + self.handle_test_discovery(discovery, event) + } + buck2_data::instant_event::Data::Snapshot(result) => { + self.handle_snapshot(result, event) + } + buck2_data::instant_event::Data::TagEvent(tag) => self.handle_tag(tag), + buck2_data::instant_event::Data::IoProviderInfo(io_provider_info) => { + self.handle_io_provider_info(io_provider_info) + } + buck2_data::instant_event::Data::TargetPatterns(tag) => { + self.handle_parsed_target_patterns(tag) + } + buck2_data::instant_event::Data::MaterializerStateInfo(materializer_state) => { + self.handle_materializer_state_info(materializer_state) } + buck2_data::instant_event::Data::StructuredError(err) => { + self.handle_structured_error(err) + } + buck2_data::instant_event::Data::RestartConfiguration(conf) => { + self.enable_restarter = conf.enable_restarter; + Ok(()) + } + buck2_data::instant_event::Data::ConcurrentCommands(concurrent_commands) => { + self.handle_concurrent_commands(concurrent_commands) + } + _ => Ok(()), } - buck2_data::buck_event::Data::Record(_) => Ok(()), } + buck2_data::buck_event::Data::Record(_) => Ok(()), } } +} - fn process_error_report(error: buck2_data::ErrorReport) -> buck2_data::ProcessedErrorReport { - buck2_data::ProcessedErrorReport { - category: error.category, - message: error.message, - telemetry_message: error.telemetry_message, - typ: error - .typ - .and_then(buck2_data::error::ErrorType::from_i32) - .map(|t| t.as_str_name().to_owned()), - source_location: error.source_location, - tags: error - .tags - .iter() - .copied() - .filter_map(buck2_data::error::ErrorTag::from_i32) - .map(|t| t.as_str_name().to_owned()) - .collect(), - } +#[allow(clippy::map_unwrap_or)] +fn process_error_report(error: buck2_data::ErrorReport) -> buck2_data::ProcessedErrorReport { + let best_tag = best_tag(error.tags.iter().filter_map(|tag| + // This should never fail, but it is safer to just ignore incorrect integers. + ErrorTag::from_i32(*tag))) + .map(|t| t.as_str_name()) + .unwrap_or(ERROR_TAG_UNCLASSIFIED); + buck2_data::ProcessedErrorReport { + category: error.category, + message: error.message, + telemetry_message: error.telemetry_message, + typ: error + .typ + .and_then(buck2_data::error::ErrorType::from_i32) + .map(|t| t.as_str_name().to_owned()), + source_location: error.source_location, + tags: error + .tags + .iter() + .copied() + .filter_map(buck2_data::error::ErrorTag::from_i32) + .map(|t| t.as_str_name().to_owned()) + .collect(), + best_tag: Some(best_tag.to_owned()), } +} - impl<'a> Drop for InvocationRecorder<'a> { - fn drop(&mut self) { - if let Some(fut) = self.send_it() { - self.async_cleanup_context - .register("sending invocation to Scribe", fut.boxed()); - } +impl<'a> Drop for InvocationRecorder<'a> { + fn drop(&mut self) { + if let Some(fut) = self.send_it() { + self.async_cleanup_context + .register("sending invocation to Scribe", fut.boxed()); } } +} - #[async_trait] - impl<'a> EventSubscriber for InvocationRecorder<'a> { - async fn handle_events(&mut self, events: &[Arc]) -> anyhow::Result<()> { - for event in events { - self.handle_event(event).await?; - } - Ok(()) +#[async_trait] +impl<'a> EventSubscriber for InvocationRecorder<'a> { + async fn handle_events(&mut self, events: &[Arc]) -> anyhow::Result<()> { + for event in events { + self.handle_event(event).await?; } + Ok(()) + } - async fn handle_console_interaction(&mut self, _c: char) -> anyhow::Result<()> { - self.tags.push("console-interaction".to_owned()); - Ok(()) - } + async fn handle_console_interaction(&mut self, _c: char) -> anyhow::Result<()> { + self.tags.push("console-interaction".to_owned()); + Ok(()) + } - async fn handle_command_result( - &mut self, - result: &buck2_cli_proto::CommandResult, - ) -> anyhow::Result<()> { - self.has_command_result = true; - match &result.result { - Some(command_result::Result::BuildResponse(res)) => { - let mut built_rule_type_names: Vec = res - .build_targets - .iter() - .map(|t| { - t.target_rule_type_name - .clone() - .unwrap_or_else(|| "NULL".to_owned()) - }) - .unique_by(|x| x.clone()) - .collect(); - built_rule_type_names.sort(); - self.target_rule_type_names = built_rule_type_names; - } - _ => {} + async fn handle_command_result( + &mut self, + result: &buck2_cli_proto::CommandResult, + ) -> anyhow::Result<()> { + self.has_command_result = true; + match &result.result { + Some(command_result::Result::BuildResponse(res)) => { + let mut built_rule_type_names: Vec = res + .build_targets + .iter() + .map(|t| { + t.target_rule_type_name + .clone() + .unwrap_or_else(|| "NULL".to_owned()) + }) + .unique_by(|x| x.clone()) + .collect(); + built_rule_type_names.sort(); + self.target_rule_type_names = built_rule_type_names; } - Ok(()) + _ => {} } + Ok(()) + } - async fn exit(&mut self) -> anyhow::Result<()> { - self.has_end_of_stream = true; - Ok(()) - } + async fn handle_error(&mut self, error: &buck2_error::Error) -> anyhow::Result<()> { + let want_stderr = error.tags().iter().any(|t| *t == ErrorTag::ClientGrpc); + let best_tag = error.best_tag(); + let error = create_error_report(error); + self.errors.push(ErrorIntermediate { + processed: process_error_report(error), + want_stderr, + best_tag, + }); + Ok(()) + } - fn as_error_observer(&self) -> Option<&dyn ErrorObserver> { - Some(self) + async fn handle_tailer_stderr(&mut self, stderr: &str) -> anyhow::Result<()> { + if self.server_stderr.len() > 100_000 { + // Proper truncation of the head is tricky, and for practical purposes + // discarding the whole thing is fine. + self.server_stderr.clear(); } - fn handle_daemon_connection_failure(&mut self, error: &buck2_error::Error) { - self.daemon_connection_failure = true; - let error = create_error_report(error); - self.errors.push(process_error_report(error)); + if !stderr.is_empty() { + // We don't know yet whether we will need stderr or not, + // so we capture it unconditionally. + self.server_stderr.push_str(stderr); + self.server_stderr.push('\n'); } + + Ok(()) } - impl<'a> ErrorObserver for InvocationRecorder<'a> { - fn daemon_in_memory_state_is_corrupted(&self) -> bool { - self.daemon_in_memory_state_is_corrupted - } + async fn exit(&mut self) -> anyhow::Result<()> { + self.has_end_of_stream = true; + Ok(()) + } - fn daemon_materializer_state_is_corrupted(&self) -> bool { - self.daemon_materializer_state_is_corrupted - } + fn as_error_observer(&self) -> Option<&dyn ErrorObserver> { + Some(self) + } - fn restarter_is_enabled(&self) -> bool { - self.enable_restarter - } + fn handle_daemon_connection_failure(&mut self, error: &buck2_error::Error) { + self.daemon_connection_failure = true; + let best_tag = error.best_tag(); + let error = create_error_report(error); + self.errors.push(ErrorIntermediate { + processed: process_error_report(error), + want_stderr: false, + best_tag, + }); } - fn calculate_diff_if_some(a: &Option, b: &Option) -> Option { - match (a, b) { - (Some(av), Some(bv)) => Some(std::cmp::max(av, bv) - std::cmp::min(av, bv)), - _ => None, - } + fn handle_daemon_started(&mut self, daemon_was_started: buck2_data::DaemonWasStartedReason) { + self.daemon_was_started = Some(daemon_was_started); } +} - fn merge_file_watcher_stats( - a: Option, - b: Option, - ) -> Option { - let (mut a, b) = match (a, b) { - (Some(a), Some(b)) => (a, b), - (a, None) => return a, - (None, b) => return b, - }; +impl<'a> ErrorObserver for InvocationRecorder<'a> { + fn daemon_in_memory_state_is_corrupted(&self) -> bool { + self.daemon_in_memory_state_is_corrupted + } + + fn daemon_materializer_state_is_corrupted(&self) -> bool { + self.daemon_materializer_state_is_corrupted + } + + fn restarter_is_enabled(&self) -> bool { + self.enable_restarter + } +} - a.fresh_instance = a.fresh_instance || b.fresh_instance; - a.events_total += b.events_total; - a.events_processed += b.events_processed; - a.branched_from_revision = a.branched_from_revision.or(b.branched_from_revision); - a.branched_from_global_rev = a.branched_from_global_rev.or(b.branched_from_global_rev); - a.events.extend(b.events); - a.incomplete_events_reason = a.incomplete_events_reason.or(b.incomplete_events_reason); - a.watchman_version = a.watchman_version.or(b.watchman_version); - Some(a) +fn calculate_diff_if_some(a: &Option, b: &Option) -> Option { + match (a, b) { + (Some(av), Some(bv)) => Some(std::cmp::max(av, bv) - std::cmp::min(av, bv)), + _ => None, } } -pub fn try_get_invocation_recorder<'a>( +fn merge_file_watcher_stats( + a: Option, + b: Option, +) -> Option { + let (mut a, b) = match (a, b) { + (Some(a), Some(b)) => (a, b), + (a, None) => return a, + (None, b) => return b, + }; + + a.fresh_instance = a.fresh_instance || b.fresh_instance; + a.events_total += b.events_total; + a.events_processed += b.events_processed; + a.branched_from_revision = a.branched_from_revision.or(b.branched_from_revision); + a.branched_from_global_rev = a.branched_from_global_rev.or(b.branched_from_global_rev); + a.events.extend(b.events); + a.incomplete_events_reason = a.incomplete_events_reason.or(b.incomplete_events_reason); + a.watchman_version = a.watchman_version.or(b.watchman_version); + Some(a) +} + +pub(crate) fn try_get_invocation_recorder<'a>( ctx: &ClientCommandContext<'a>, opts: &CommonDaemonCommandOptions, command_name: &'static str, sanitized_argv: Vec, log_size_counter_bytes: Option>, -) -> anyhow::Result>> { +) -> anyhow::Result>> { let write_to_path = opts .unstable_write_invocation_record .as_ref() @@ -1169,7 +1270,7 @@ pub fn try_get_invocation_recorder<'a>( filesystem = "default".to_owned(); } - let recorder = imp::InvocationRecorder::new( + let recorder = InvocationRecorder::new( ctx.fbinit(), ctx.async_cleanup_context().dupe(), write_to_path, @@ -1189,23 +1290,29 @@ pub fn try_get_invocation_recorder<'a>( Ok(Box::new(recorder)) } -fn system_memory_stats() -> u64 { - use sysinfo::RefreshKind; - use sysinfo::System; - use sysinfo::SystemExt; - - let system = System::new_with_specifics(RefreshKind::new().with_memory()); - system.total_memory() +fn truncate_stderr(stderr: &str) -> &str { + // If server crashed, it means something is very broken, + // and we don't really need nicely formatted stderr. + // We only need to see it once, fix it, and never see it again. + let max_len = 20_000; + let truncate_at = stderr.len().saturating_sub(max_len); + let truncate_at = stderr.ceil_char_boundary(truncate_at); + &stderr[truncate_at..] } #[cfg(test)] mod tests { - use super::*; + use crate::subscribers::recorder::truncate_stderr; #[test] - fn get_system_memory_stats() { - let total_mem = system_memory_stats(); - // sysinfo returns zero when fails to retrieve data - assert!(total_mem > 0); + fn test_truncate_stderr() { + let mut stderr = String::new(); + stderr.push_str("prefix"); + stderr.push('Ъ'); // 2 bytes, so asking to truncate in the middle of the char. + for _ in 0..19_999 { + stderr.push('a'); + } + let truncated = truncate_stderr(&stderr); + assert_eq!(truncated.len(), 19_999); } } diff --git a/app/buck2_client_ctx/src/subscribers/simpleconsole.rs b/app/buck2_client_ctx/src/subscribers/simpleconsole.rs index 026b5f254bb99..658f1f5ba8038 100644 --- a/app/buck2_client_ctx/src/subscribers/simpleconsole.rs +++ b/app/buck2_client_ctx/src/subscribers/simpleconsole.rs @@ -543,7 +543,7 @@ where Ok(()) } - async fn handle_error(&mut self, _error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, _error: &buck2_error::Error) -> anyhow::Result<()> { // We don't need to do any cleanup to exit. Ok(()) } diff --git a/app/buck2_client_ctx/src/subscribers/subscriber.rs b/app/buck2_client_ctx/src/subscribers/subscriber.rs index a7d5d3320b358..8865cc2b7fd65 100644 --- a/app/buck2_client_ctx/src/subscribers/subscriber.rs +++ b/app/buck2_client_ctx/src/subscribers/subscriber.rs @@ -63,7 +63,7 @@ pub trait EventSubscriber: Send { /// Give the subscriber a chance to react to errors as we start trying to clean up. /// They may return another error, which will be incorporated into the end result. - async fn handle_error(&mut self, _error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, _error: &buck2_error::Error) -> anyhow::Result<()> { Ok(()) } @@ -82,4 +82,5 @@ pub trait EventSubscriber: Send { } fn handle_daemon_connection_failure(&mut self, _error: &buck2_error::Error) {} + fn handle_daemon_started(&mut self, _reason: buck2_data::DaemonWasStartedReason) {} } diff --git a/app/buck2_client_ctx/src/subscribers/subscriber_unpack.rs b/app/buck2_client_ctx/src/subscribers/subscriber_unpack.rs index 10661a8769827..ef645a6cad24b 100644 --- a/app/buck2_client_ctx/src/subscribers/subscriber_unpack.rs +++ b/app/buck2_client_ctx/src/subscribers/subscriber_unpack.rs @@ -221,7 +221,7 @@ pub trait UnpackingEventSubscriber: Send { /// Give the subscriber a chance to react to errors as we start trying to clean up. /// They may return another error, which will be incorporated into the end result. - async fn handle_error(&mut self, _error: &anyhow::Error) -> anyhow::Result<()>; + async fn handle_error(&mut self, _error: &buck2_error::Error) -> anyhow::Result<()>; /// Allow the subscriber to do some sort of action once every render cycle. async fn tick(&mut self, _tick: &Tick) -> anyhow::Result<()>; @@ -255,7 +255,7 @@ impl EventSubscriber for UnpackingEventSubscriberAs Ok(()) } - async fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, error: &buck2_error::Error) -> anyhow::Result<()> { self.0.handle_error(error).await } diff --git a/app/buck2_client_ctx/src/subscribers/subscribers.rs b/app/buck2_client_ctx/src/subscribers/subscribers.rs new file mode 100644 index 0000000000000..a59c04caddf5a --- /dev/null +++ b/app/buck2_client_ctx/src/subscribers/subscribers.rs @@ -0,0 +1,82 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::future::Future; + +use futures::stream::FuturesUnordered; +use futures::StreamExt; + +use crate::subscribers::observer::ErrorObserver; +use crate::subscribers::subscriber::EventSubscriber; + +#[derive(Default)] +pub struct EventSubscribers<'a> { + subscribers: Vec>, +} + +impl<'a> EventSubscribers<'a> { + pub fn new(subscribers: Vec>) -> EventSubscribers<'a> { + EventSubscribers { subscribers } + } + + /// Helper method to abstract the process of applying an `EventSubscriber` method to all of the subscribers. + /// Quits on the first error encountered. + pub(crate) async fn for_each_subscriber<'b, Fut>( + &'b mut self, + f: impl FnMut(&'b mut Box) -> Fut, + ) -> anyhow::Result<()> + where + Fut: Future> + 'b, + { + let mut futures: FuturesUnordered<_> = self.subscribers.iter_mut().map(f).collect(); + while let Some(res) = futures.next().await { + res?; + } + Ok(()) + } + + pub(crate) async fn handle_exit(&mut self) -> anyhow::Result<()> { + let mut r = Ok(()); + for subscriber in &mut self.subscribers { + // Exit all subscribers, do not stop on first one. + let subscriber_err = subscriber.exit().await; + if r.is_ok() { + // Keep first error. + r = subscriber_err; + } + } + r + } + + pub(crate) fn handle_daemon_connection_failure(&mut self, error: &buck2_error::Error) { + for subscriber in &mut self.subscribers { + subscriber.handle_daemon_connection_failure(error); + } + } + + pub(crate) fn handle_daemon_started(&mut self, reason: buck2_data::DaemonWasStartedReason) { + for subscriber in &mut self.subscribers { + subscriber.handle_daemon_started(reason); + } + } + + pub(crate) fn error_observers(&self) -> impl Iterator { + self.subscribers + .iter() + .filter_map(|s| s.as_error_observer()) + } + + pub(crate) async fn eprintln(&mut self, message: &str) -> anyhow::Result<()> { + self.for_each_subscriber(|s| { + // TODO(nga): this is not a tailer. + s.handle_tailer_stderr(message) + }) + .await + } +} diff --git a/app/buck2_client_ctx/src/subscribers/superconsole.rs b/app/buck2_client_ctx/src/subscribers/superconsole.rs index c2a5a8e5bb99d..3734d65ceae7c 100644 --- a/app/buck2_client_ctx/src/subscribers/superconsole.rs +++ b/app/buck2_client_ctx/src/subscribers/superconsole.rs @@ -566,7 +566,7 @@ impl UnpackingEventSubscriber for StatefulSuperConsole { } } - async fn handle_error(&mut self, _error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, _error: &buck2_error::Error) -> anyhow::Result<()> { match self.super_console.take() { Some(super_console) => super_console.finalize(&BuckRootComponent { header: &self.header, diff --git a/app/buck2_common/BUCK b/app/buck2_common/BUCK index d97dde9b36316..0a86df7a32cf4 100644 --- a/app/buck2_common/BUCK +++ b/app/buck2_common/BUCK @@ -35,6 +35,7 @@ rust_library( ], deps = [ "fbsource//third-party/rust:anyhow", + "fbsource//third-party/rust:async-scoped", "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:blake3", "fbsource//third-party/rust:chrono", diff --git a/app/buck2_common/Cargo.toml b/app/buck2_common/Cargo.toml index dd54594a6b363..c9ac6832f1132 100644 --- a/app/buck2_common/Cargo.toml +++ b/app/buck2_common/Cargo.toml @@ -7,6 +7,7 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } +async-scoped = { workspace = true } async-trait = { workspace = true } blake3 = { workspace = true } chrono = { workspace = true } diff --git a/app/buck2_common/src/file_ops.rs b/app/buck2_common/src/file_ops.rs index 9978897c0114d..3b535eaaed316 100644 --- a/app/buck2_common/src/file_ops.rs +++ b/app/buck2_common/src/file_ops.rs @@ -220,7 +220,7 @@ impl FileDigest { /// Stores the relevant metadata for a file. // New fields should be added as needed, and unused fields removed. #[derive(Debug, Dupe, Hash, PartialEq, Eq, Clone, Display, Allocative)] -#[display(fmt = "File({})", digest)] +#[display(fmt = "File(digest={}, is_executable={})", digest, is_executable)] pub struct FileMetadata { pub digest: TrackedFileDigest, pub is_executable: bool, diff --git a/app/buck2_common/src/global_cfg_options.rs b/app/buck2_common/src/global_cfg_options.rs new file mode 100644 index 0000000000000..3cb027018b3c2 --- /dev/null +++ b/app/buck2_common/src/global_cfg_options.rs @@ -0,0 +1,22 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::sync::Arc; + +use allocative::Allocative; +use buck2_core::target::label::TargetLabel; +use dupe::Dupe; + +#[derive( + Default, Debug, Dupe, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Allocative +)] +pub struct GlobalCfgOptions { + pub target_platform: Option, + pub cli_modifiers: Arc>, +} diff --git a/app/buck2_common/src/invocation_paths.rs b/app/buck2_common/src/invocation_paths.rs index 5f743ac8c8b70..af3bbdcf1f696 100644 --- a/app/buck2_common/src/invocation_paths.rs +++ b/app/buck2_common/src/invocation_paths.rs @@ -9,7 +9,6 @@ //! //! Defines utilities to obtain the basic paths for buck2 client and the daemon. -//! use std::borrow::Cow; diff --git a/app/buck2_common/src/io/trace.rs b/app/buck2_common/src/io/trace.rs index b823056a56626..481b347f95e9e 100644 --- a/app/buck2_common/src/io/trace.rs +++ b/app/buck2_common/src/io/trace.rs @@ -82,6 +82,10 @@ impl TracingIoProvider { } } + pub fn from_io(io: &dyn IoProvider) -> Option<&Self> { + io.as_any().downcast_ref::() + } + pub fn add_project_path(&self, path: ProjectRelativePathBuf) { self.trace.project_entries.insert(path); } diff --git a/app/buck2_common/src/legacy_configs/init.rs b/app/buck2_common/src/legacy_configs/init.rs index 3029ee6d14556..76c0d846520d9 100644 --- a/app/buck2_common/src/legacy_configs/init.rs +++ b/app/buck2_common/src/legacy_configs/init.rs @@ -51,6 +51,7 @@ pub struct HttpConfig { connect_timeout_ms: Option, read_timeout_ms: Option, write_timeout_ms: Option, + pub http2: bool, pub max_redirects: Option, } @@ -60,12 +61,14 @@ impl HttpConfig { let read_timeout_ms = config.parse("http", "read_timeout_ms")?; let write_timeout_ms = config.parse("http", "write_timeout_ms")?; let max_redirects = config.parse("http", "max_redirects")?; + let http2 = config.parse("http", "http2")?.unwrap_or(true); Ok(Self { connect_timeout_ms, read_timeout_ms, write_timeout_ms, max_redirects, + http2, }) } diff --git a/app/buck2_common/src/legacy_configs/view.rs b/app/buck2_common/src/legacy_configs/view.rs index 0c7309164f2de..1d0302e0a2397 100644 --- a/app/buck2_common/src/legacy_configs/view.rs +++ b/app/buck2_common/src/legacy_configs/view.rs @@ -8,10 +8,13 @@ */ use std::fmt::Debug; +use std::str::FromStr; use std::sync::Arc; use buck2_core::cells::name::CellName; +use crate::legacy_configs::LegacyBuckConfig; + /// Buckconfig trait. /// /// There are two implementations: @@ -21,6 +24,17 @@ pub trait LegacyBuckConfigView: Debug { fn get(&self, section: &str, key: &str) -> anyhow::Result>>; } +impl dyn LegacyBuckConfigView + '_ { + pub fn parse(&self, section: &str, key: &str) -> anyhow::Result> + where + anyhow::Error: From<::Err>, + { + self.get(section, key)? + .map(|s| LegacyBuckConfig::parse_impl(section, key, &s)) + .transpose() + } +} + /// All cell buckconfigs traits. pub trait LegacyBuckConfigsView { fn get<'a>(&'a self, cell_name: CellName) -> anyhow::Result<&'a dyn LegacyBuckConfigView>; diff --git a/app/buck2_common/src/lib.rs b/app/buck2_common/src/lib.rs index 34195f84fb3cd..0bf1f39d747e3 100644 --- a/app/buck2_common/src/lib.rs +++ b/app/buck2_common/src/lib.rs @@ -32,6 +32,7 @@ pub mod events; pub mod external_symlink; pub mod file_ops; pub mod find_buildfile; +pub mod global_cfg_options; pub mod home_buck_tmp; pub mod http; pub mod ignores; @@ -46,6 +47,7 @@ pub mod memory; pub mod package_boundary; pub mod package_listing; pub mod pattern; +pub mod scope; pub mod sqlite; pub mod target_aliases; pub mod temp_path; diff --git a/app/buck2_common/src/liveliness_observer.rs b/app/buck2_common/src/liveliness_observer.rs index 4a40c1ea2182a..28648dc1a2f98 100644 --- a/app/buck2_common/src/liveliness_observer.rs +++ b/app/buck2_common/src/liveliness_observer.rs @@ -8,11 +8,15 @@ */ use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use dupe::Dupe; +use futures::future::FutureExt; +use futures::future::Shared; use tokio::sync::OwnedRwLockWriteGuard; use tokio::sync::RwLock; +use tokio::time::Sleep; #[derive(Debug, buck2_error::Error, Copy, Clone, Dupe)] #[error("LivelinessObserver reports this session is shutting down")] @@ -196,6 +200,25 @@ impl LivelinessObserver for buck2_futures::cancellable_future::CancellationObser } } +pub struct TimeoutLivelinessObserver { + inner: Shared, +} + +impl TimeoutLivelinessObserver { + pub fn new(duration: Duration) -> Self { + Self { + inner: tokio::time::sleep(duration).shared(), + } + } +} + +#[async_trait] +impl LivelinessObserver for TimeoutLivelinessObserver { + async fn while_alive(&self) { + self.inner.clone().await + } +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +258,19 @@ mod tests { restored.forget(); assert!(manager.is_alive().await); } + + #[tokio::test] + async fn test_timeout() { + let obs = TimeoutLivelinessObserver::new(Duration::from_secs(1)); + + // It is alive for a little while. + tokio::time::timeout(Duration::from_millis(100), obs.while_alive()) + .await + .unwrap_err(); + + // It eventually becomes not-alive. + tokio::time::timeout(Duration::from_secs(10), obs.while_alive()) + .await + .unwrap(); + } } diff --git a/app/buck2_common/src/package_listing/file_listing.rs b/app/buck2_common/src/package_listing/file_listing.rs index 3d6218fae3131..fe1443e95f92e 100644 --- a/app/buck2_common/src/package_listing/file_listing.rs +++ b/app/buck2_common/src/package_listing/file_listing.rs @@ -105,7 +105,7 @@ pub mod testing { } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_common/src/scope.rs b/app/buck2_common/src/scope.rs new file mode 100644 index 0000000000000..d3e69d2c40dcf --- /dev/null +++ b/app/buck2_common/src/scope.rs @@ -0,0 +1,76 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::future::Future; + +use buck2_events::dispatch::with_dispatcher_async; +use buck2_events::dispatch::EventDispatcher; +use dice::DiceComputations; +use dupe::Dupe; + +use crate::events::HasEvents; + +pub struct Scope<'a, 'x, T> +where + T: Send + 'static, + 'a: 'x, +{ + scope: &'x mut async_scoped::TokioScope<'a, T>, + dispatcher: EventDispatcher, +} + +impl<'a, 'x, T> Scope<'a, 'x, T> +where + T: Send + 'static, + 'a: 'x, +{ + pub fn spawn_cancellable + Send + 'a, Fu: FnOnce() -> T + Send + 'a>( + &mut self, + f: F, + default: Fu, + ) { + self.scope + .spawn_cancellable(with_dispatcher_async(self.dispatcher.dupe(), f), default) + } +} + +/// Wrap `async_scoped::TokioScope::scope_and_collect` propagating the event dispatcher. +pub async unsafe fn scope_and_collect_with_dispatcher<'d, 'a, T, R, F>( + dispatcher: EventDispatcher, + f: F, +) -> ( + R, + Vec<>::FutureOutput>, +) + where + T: Send + 'static, + F: for<'x> FnOnce(&mut Scope<'a, 'x, T>) -> R, +{ + async_scoped::TokioScope::scope_and_collect(|scope| { + let mut scope = Scope { scope, dispatcher }; + f(&mut scope) + }) + .await +} + +/// Wrap `async_scoped::TokioScope::scope_and_collect` propagating the event dispatcher. +pub async unsafe fn scope_and_collect_with_dice<'d, 'a, T, R, F>( + ctx: &'d mut DiceComputations, + f: F, +) -> ( + R, + Vec<>::FutureOutput>, +) +where + T: Send + 'static, + F: for<'x> FnOnce(&'d mut DiceComputations, &mut Scope<'a, 'x, T>) -> R, +{ + let dispatcher = ctx.per_transaction_data().get_dispatcher().dupe(); + scope_and_collect_with_dispatcher(dispatcher, |scope| f(ctx, scope)).await +} diff --git a/app/buck2_configured/BUCK b/app/buck2_configured/BUCK index 9961871b19ddf..3ef5431aaa0a9 100644 --- a/app/buck2_configured/BUCK +++ b/app/buck2_configured/BUCK @@ -13,7 +13,6 @@ rust_library( "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:derive_more", "fbsource//third-party/rust:futures", - "fbsource//third-party/rust:indexmap", "//buck2/allocative/allocative:allocative", "//buck2/app/buck2_build_api:buck2_build_api", "//buck2/app/buck2_common:buck2_common", diff --git a/app/buck2_configured/Cargo.toml b/app/buck2_configured/Cargo.toml index 091d8d1072062..19bc7f995f712 100644 --- a/app/buck2_configured/Cargo.toml +++ b/app/buck2_configured/Cargo.toml @@ -10,7 +10,6 @@ anyhow = { workspace = true } async-trait = { workspace = true } derive_more = { workspace = true } futures = { workspace = true } -indexmap = { workspace = true } allocative = { workspace = true } dice = { workspace = true } diff --git a/app/buck2_configured/src/calculation.rs b/app/buck2_configured/src/calculation.rs index b9a504d5dca70..7c9d2ab365719 100644 --- a/app/buck2_configured/src/calculation.rs +++ b/app/buck2_configured/src/calculation.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use async_trait::async_trait; use buck2_common::dice::cycles::CycleAdapterDescriptor; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::configuration::data::ConfigurationData; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_core::target::label::TargetLabel; @@ -28,6 +29,7 @@ use gazebo::prelude::*; use crate::configuration::calculation::ConfigurationCalculation; use crate::nodes::calculation::get_execution_platform_toolchain_dep; use crate::nodes::calculation::ConfiguredTargetNodeKey; +use crate::target::TargetConfiguredTargetLabel; struct ConfiguredTargetCalculationInstance; @@ -41,12 +43,12 @@ impl ConfiguredTargetCalculationImpl for ConfiguredTargetCalculationInstance { &self, ctx: &DiceComputations, target: &TargetLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result { let (node, super_package) = ctx.get_target_node_with_super_package(target).await?; let get_platform_configuration = async || -> buck2_error::Result { - let current_cfg = match global_target_platform { + let current_cfg = match global_cfg_options.target_platform.as_ref() { Some(global_target_platform) => { ctx.get_platform_configuration(global_target_platform) .await? @@ -59,7 +61,14 @@ impl ConfiguredTargetCalculationImpl for ConfiguredTargetCalculationInstance { Ok(CFG_CONSTRUCTOR_CALCULATION_IMPL .get()? - .eval_cfg_constructor(ctx, &node, &super_package, current_cfg) + .eval_cfg_constructor( + ctx, + node.as_ref(), + &super_package, + current_cfg, + &global_cfg_options.cli_modifiers, + node.rule_type(), + ) .await?) }; @@ -68,11 +77,14 @@ impl ConfiguredTargetCalculationImpl for ConfiguredTargetCalculationInstance { RuleKind::Normal => Ok(target.configure(get_platform_configuration().await?)), RuleKind::Toolchain => { let cfg = get_platform_configuration().await?; - let exec_cfg = - get_execution_platform_toolchain_dep(ctx, &target.configure(cfg.dupe()), &node) - .await? - .require_compatible()? - .cfg(); + let exec_cfg = get_execution_platform_toolchain_dep( + ctx, + &TargetConfiguredTargetLabel::new_configure(target, cfg.dupe()), + node.as_ref(), + ) + .await? + .require_compatible()? + .cfg(); Ok(target.configure_with_exec(cfg, exec_cfg.cfg().dupe())) } } diff --git a/app/buck2_configured/src/configuration/calculation.rs b/app/buck2_configured/src/configuration/calculation.rs index 50773a3f0cba9..e70baf4650a2c 100644 --- a/app/buck2_configured/src/configuration/calculation.rs +++ b/app/buck2_configured/src/configuration/calculation.rs @@ -9,7 +9,7 @@ use std::sync::Arc; -use allocative::Allocative;use starlark_map::small_map::SmallMap; +use allocative::Allocative; use async_trait::async_trait;use buck2_build_api::interpreter::rule_defs::provider::builtin::platform_info::FrozenPlatformInfo; use buck2_common::dice::cells::HasCellResolver; use buck2_common::legacy_configs::dice::HasLegacyConfigs; @@ -21,7 +21,6 @@ use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::configuration::config_setting::ConfigSettingData; use buck2_core::configuration::data::ConfigurationData; use buck2_core::configuration::pair::ConfigurationNoExec; -use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_core::target::label::TargetLabel; use buck2_core::execution_types::execution::ExecutionPlatform; use buck2_core::execution_types::execution::ExecutionPlatformError; @@ -41,7 +40,6 @@ use dice::DiceComputations; use dice::Key; use dupe::Dupe; use gazebo::prelude::*; -use indexmap::IndexSet; use buck2_futures::cancellation::CancellationContext; use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; use buck2_build_api::interpreter::rule_defs::provider::builtin::configuration_info::FrozenConfigurationInfo; @@ -160,29 +158,27 @@ async fn check_execution_platform( ctx: &DiceComputations, target_node_cell: CellName, exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, + exec_deps: &[TargetLabel], exec_platform: &ExecutionPlatform, toolchain_allows: &[ToolchainConstraints], ) -> anyhow::Result> { - // First check if the platform satisfies the toolchain requirements - for allowed in toolchain_allows { - if let Err(e) = allowed.allows(exec_platform) { - return Ok(Err( - ExecutionPlatformIncompatibleReason::ExecutionDependencyIncompatible(e), - )); - } - } - let resolved_platform_configuration = ctx .get_resolved_configuration( exec_platform.cfg(), target_node_cell, - exec_compatible_with.iter(), + toolchain_allows + .iter() + .flat_map(ToolchainConstraints::exec_compatible_with) + .chain(exec_compatible_with), ) .await?; // Then check if the platform satisfies compatible_with - for constraint in exec_compatible_with { + for constraint in toolchain_allows + .iter() + .flat_map(ToolchainConstraints::exec_compatible_with) + .chain(exec_compatible_with) + { if resolved_platform_configuration .matches(constraint) .is_none() @@ -196,7 +192,11 @@ async fn check_execution_platform( // Then check that all exec_deps are compatible with the platform. We collect errors separately, // so that we do not report an error if we would later find an incompatibility let mut errs = Vec::new(); - for dep in exec_deps { + for dep in toolchain_allows + .iter() + .flat_map(ToolchainConstraints::exec_deps) + .chain(exec_deps) + { match ctx .get_configured_target_node( &dep.configure_pair_no_exec(exec_platform.cfg_pair_no_exec().dupe()), @@ -233,39 +233,11 @@ async fn get_execution_platforms_enabled( .context("Execution platforms are not enabled") } -pub(crate) async fn resolve_toolchain_constraints_from_constraints( - ctx: &DiceComputations, - target: &ConfiguredTargetLabel, - exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, - toolchain_allows: &[ToolchainConstraints], -) -> buck2_error::Result { - let mut incompatible = SmallMap::new(); - for exec_platform in get_execution_platforms_enabled(ctx).await?.candidates() { - if let Err(e) = check_execution_platform( - ctx, - target.pkg().cell_name(), - exec_compatible_with, - exec_deps, - exec_platform, - toolchain_allows, - ) - .await? - { - incompatible.insert( - exec_platform.dupe(), - Arc::new(e.into_incompatible_platform_reason(target.dupe())), - ); - } - } - Ok(ToolchainConstraints::new(incompatible)) -} - async fn resolve_execution_platform_from_constraints( ctx: &DiceComputations, target_node_cell: CellName, exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, + exec_deps: &[TargetLabel], toolchain_allows: &[ToolchainConstraints], ) -> buck2_error::Result { let mut skipped = Vec::new(); @@ -409,18 +381,10 @@ pub trait ConfigurationCalculation { async fn resolve_execution_platform_from_constraints( &self, target_node_cell: CellName, - exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, - toolchain_allows: &[ToolchainConstraints], + exec_compatible_with: Arc<[TargetLabel]>, + exec_deps: Arc<[TargetLabel]>, + toolchain_allows: Arc<[ToolchainConstraints]>, ) -> buck2_error::Result; - - async fn resolve_toolchain_constraints_from_constraints( - &self, - target: &ConfiguredTargetLabel, - exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, - toolchain_allows: &[ToolchainConstraints], - ) -> buck2_error::Result; } async fn compute_platform_configuration_no_label_check( @@ -593,13 +557,12 @@ impl ConfigurationCalculation for DiceComputations { { Some(configuration_info) => configuration_info, None => { - return Err::<_, anyhow::Error>( + return Err::<_, buck2_error::Error>( ConfigurationError::MissingConfigurationInfoProvider( self.cfg_target.dupe(), ) .into(), - ) - .map_err(buck2_error::Error::from); + ); } } .to_config_setting_data(); @@ -662,34 +625,45 @@ impl ConfigurationCalculation for DiceComputations { async fn resolve_execution_platform_from_constraints( &self, target_node_cell: CellName, - exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, - toolchain_allows: &[ToolchainConstraints], + exec_compatible_with: Arc<[TargetLabel]>, + exec_deps: Arc<[TargetLabel]>, + toolchain_allows: Arc<[ToolchainConstraints]>, ) -> buck2_error::Result { - resolve_execution_platform_from_constraints( - self, - target_node_cell, - exec_compatible_with, - exec_deps, - toolchain_allows, - ) - .await - } + #[derive(Clone, Dupe, Display, Debug, Eq, Hash, PartialEq, Allocative)] + #[display(fmt = "{:?}", self)] + struct ExecutionPlatformResolutionKey( + CellName, + Arc<[TargetLabel]>, + Arc<[TargetLabel]>, + Arc<[ToolchainConstraints]>, + ); - async fn resolve_toolchain_constraints_from_constraints( - &self, - target: &ConfiguredTargetLabel, - exec_compatible_with: &[TargetLabel], - exec_deps: &IndexSet, - toolchain_allows: &[ToolchainConstraints], - ) -> buck2_error::Result { - resolve_toolchain_constraints_from_constraints( - self, - target, + #[async_trait] + impl Key for ExecutionPlatformResolutionKey { + type Value = buck2_error::Result; + + async fn compute( + &self, + ctx: &mut DiceComputations, + _cancellation: &CancellationContext, + ) -> Self::Value { + resolve_execution_platform_from_constraints(ctx, self.0, &self.1, &self.2, &self.3) + .await + } + + fn equality(x: &Self::Value, y: &Self::Value) -> bool { + match (x, y) { + (Ok(x), Ok(y)) => x == y, + _ => false, + } + } + } + self.compute(&ExecutionPlatformResolutionKey( + target_node_cell, exec_compatible_with, exec_deps, toolchain_allows, - ) - .await + )) + .await? } } diff --git a/app/buck2_configured/src/lib.rs b/app/buck2_configured/src/lib.rs index 02ff05f5d5101..8048fe5c34218 100644 --- a/app/buck2_configured/src/lib.rs +++ b/app/buck2_configured/src/lib.rs @@ -13,6 +13,7 @@ pub mod calculation; pub mod configuration; pub mod nodes; +pub mod target; pub fn init_late_bindings() { calculation::init_configured_target_calculation(); diff --git a/app/buck2_configured/src/nodes/calculation.rs b/app/buck2_configured/src/nodes/calculation.rs index 6065aa7f25a76..6eead7d5e73ba 100644 --- a/app/buck2_configured/src/nodes/calculation.rs +++ b/app/buck2_configured/src/nodes/calculation.rs @@ -56,19 +56,19 @@ use buck2_node::nodes::configured_frontend::ConfiguredTargetNodeCalculationImpl; use buck2_node::nodes::configured_frontend::CONFIGURED_TARGET_NODE_CALCULATION; use buck2_node::nodes::frontend::TargetGraphCalculation; use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeRef; use buck2_node::visibility::VisibilityError; use derive_more::Display; use dice::DiceComputations; use dice::Key; use dupe::Dupe; -use indexmap::IndexSet; use starlark_map::ordered_map::OrderedMap; use starlark_map::small_map::SmallMap; use starlark_map::small_set::SmallSet; use crate::calculation::ConfiguredGraphCycleDescriptor; -use crate::configuration::calculation::resolve_toolchain_constraints_from_constraints; use crate::configuration::calculation::ConfigurationCalculation; +use crate::target::TargetConfiguredTargetLabel; #[derive(Debug, buck2_error::Error)] enum NodeCalculationError { @@ -89,7 +89,7 @@ enum CompatibilityConstraints { async fn compute_platform_cfgs( ctx: &DiceComputations, - node: &TargetNode, + node: TargetNodeRef<'_>, ) -> anyhow::Result> { let mut platform_map = OrderedMap::new(); for platform_target in node.platform_deps() { @@ -148,18 +148,17 @@ pub async fn find_execution_platform_by_configuration( } } -#[derive(Default)] pub struct ExecutionPlatformConstraints { - exec_deps: IndexSet, - toolchain_deps: IndexSet, - exec_compatible_with: Vec, + exec_deps: Arc<[TargetLabel]>, + toolchain_deps: Arc<[TargetConfiguredTargetLabel]>, + exec_compatible_with: Arc<[TargetLabel]>, } impl ExecutionPlatformConstraints { pub fn new_constraints( - exec_deps: IndexSet, - toolchain_deps: IndexSet, - exec_compatible_with: Vec, + exec_deps: Arc<[TargetLabel]>, + toolchain_deps: Arc<[TargetConfiguredTargetLabel]>, + exec_compatible_with: Arc<[TargetLabel]>, ) -> Self { Self { exec_deps, @@ -169,12 +168,11 @@ impl ExecutionPlatformConstraints { } fn new( - node: &TargetNode, + node: TargetNodeRef, gathered_deps: &GatheredDeps, cfg_ctx: &(dyn AttrConfigurationContext + Sync), ) -> buck2_error::Result { - let mut exec_compatible_with = Vec::new(); - if let Some(a) = node.attr_or_none( + let exec_compatible_with: Arc<[_]> = if let Some(a) = node.attr_or_none( EXEC_COMPATIBLE_WITH_ATTRIBUTE_FIELD, AttrInspectOptions::All, ) { @@ -184,13 +182,16 @@ impl ExecutionPlatformConstraints { a.name ) })?; - for label in ConfiguredTargetNode::attr_as_target_compatible_with(configured_attr.value) - { - exec_compatible_with.push(label.with_context(|| { - format!("attribute `{}`", EXEC_COMPATIBLE_WITH_ATTRIBUTE_FIELD) - })?); - } - } + ConfiguredTargetNode::attr_as_target_compatible_with(configured_attr.value) + .map(|label| { + label.with_context(|| { + format!("attribute `{}`", EXEC_COMPATIBLE_WITH_ATTRIBUTE_FIELD) + }) + }) + .collect::>()? + } else { + Arc::new([]) + }; Ok(Self::new_constraints( gathered_deps @@ -210,56 +211,43 @@ impl ExecutionPlatformConstraints { async fn toolchain_allows( &self, ctx: &DiceComputations, - ) -> buck2_error::Result> { + ) -> buck2_error::Result> { // We could merge these constraints together, but the time to do that // probably outweighs the benefits given there are likely to only be a few // execution platforms to test. let mut result = Vec::with_capacity(self.toolchain_deps.len()); - for x in &self.toolchain_deps { + for x in self.toolchain_deps.iter() { result.push(execution_platforms_for_toolchain(ctx, x.dupe()).await?) } - Ok(result) + Ok(result.into()) } async fn one( - &self, + self, ctx: &DiceComputations, - node: &TargetNode, + node: TargetNodeRef<'_>, ) -> buck2_error::Result { + let toolchain_allows = self.toolchain_allows(ctx).await?; ctx.resolve_execution_platform_from_constraints( node.label().pkg().cell_name(), - &self.exec_compatible_with, - &self.exec_deps, - &self.toolchain_allows(ctx).await?, + self.exec_compatible_with, + self.exec_deps, + toolchain_allows, ) .await } pub async fn one_for_cell( - &self, + self, ctx: &DiceComputations, cell: CellName, ) -> buck2_error::Result { + let toolchain_allows = self.toolchain_allows(ctx).await?; ctx.resolve_execution_platform_from_constraints( cell, - &self.exec_compatible_with, - &self.exec_deps, - &self.toolchain_allows(ctx).await?, - ) - .await - } - - async fn many( - &self, - ctx: &DiceComputations, - target: &ConfiguredTargetLabel, - ) -> buck2_error::Result { - resolve_toolchain_constraints_from_constraints( - ctx, - target, - &self.exec_compatible_with, - &self.exec_deps, - &self.toolchain_allows(ctx).await?, + self.exec_compatible_with, + self.exec_deps, + toolchain_allows, ) .await } @@ -267,10 +255,10 @@ impl ExecutionPlatformConstraints { async fn execution_platforms_for_toolchain( ctx: &DiceComputations, - target: ConfiguredTargetLabel, + target: TargetConfiguredTargetLabel, ) -> buck2_error::Result { #[derive(Clone, Display, Debug, Dupe, Eq, Hash, PartialEq, Allocative)] - struct ExecutionPlatformsForToolchainKey(ConfiguredTargetLabel); + struct ExecutionPlatformsForToolchainKey(TargetConfiguredTargetLabel); #[async_trait] impl Key for ExecutionPlatformsForToolchainKey { @@ -295,7 +283,7 @@ async fn execution_platforms_for_toolchain( node.get_configuration_deps(), ) .await?; - let platform_cfgs = compute_platform_cfgs(ctx, &node).await?; + let platform_cfgs = compute_platform_cfgs(ctx, node.as_ref()).await?; // We don't really need `resolved_transitions` here: // `Traversal` declared above ignores transitioned dependencies. // But we pass `resolved_transitions` here to prevent breakages in the future @@ -308,14 +296,20 @@ async fn execution_platforms_for_toolchain( &platform_cfgs, ); let (gathered_deps, errors_and_incompats) = - gather_deps(&self.0, &node, &cfg_ctx, ctx).await?; + gather_deps(&self.0, node.as_ref(), &cfg_ctx, ctx).await?; if let Some(ret) = errors_and_incompats.finalize() { // Statically assert that we hit one of the `?`s enum Void {} let _: Void = ret?.require_compatible()?; } - let constraints = ExecutionPlatformConstraints::new(&node, &gathered_deps, &cfg_ctx)?; - constraints.many(ctx, &self.0).await + let constraints = + ExecutionPlatformConstraints::new(node.as_ref(), &gathered_deps, &cfg_ctx)?; + let toolchain_allows = constraints.toolchain_allows(ctx).await?; + Ok(ToolchainConstraints::new( + &constraints.exec_deps, + &constraints.exec_compatible_with, + &toolchain_allows, + )) } fn equality(x: &Self::Value, y: &Self::Value) -> bool { @@ -332,8 +326,8 @@ async fn execution_platforms_for_toolchain( pub async fn get_execution_platform_toolchain_dep( ctx: &DiceComputations, - target_label: &ConfiguredTargetLabel, - target_node: &TargetNode, + target_label: &TargetConfiguredTargetLabel, + target_node: TargetNodeRef<'_>, ) -> buck2_error::Result> { assert!(target_node.is_toolchain_rule()); let target_cfg = target_label.cfg(); @@ -378,7 +372,7 @@ pub async fn get_execution_platform_toolchain_dep( async fn resolve_execution_platform( ctx: &DiceComputations, - node: &TargetNode, + node: TargetNodeRef<'_>, resolved_configuration: &ResolvedConfiguration, gathered_deps: &GatheredDeps, cfg_ctx: &(dyn AttrConfigurationContext + Sync), @@ -401,7 +395,7 @@ async fn resolve_execution_platform( } fn unpack_target_compatible_with_attr( - target_node: &TargetNode, + target_node: TargetNodeRef, resolved_cfg: &ResolvedConfiguration, attr_name: &str, ) -> anyhow::Result> { @@ -476,7 +470,7 @@ fn unpack_target_compatible_with_attr( fn check_compatible( target_label: &ConfiguredTargetLabel, - target_node: &TargetNode, + target_node: TargetNodeRef, resolved_cfg: &ResolvedConfiguration, ) -> anyhow::Result> { let target_compatible_with = unpack_target_compatible_with_attr( @@ -603,7 +597,7 @@ struct ErrorsAndIncompatibilities { impl ErrorsAndIncompatibilities { pub fn unpack_dep_into( &mut self, - target_label: &ConfiguredTargetLabel, + target_label: &TargetConfiguredTargetLabel, result: anyhow::Result>, check_visibility: CheckVisibility, list: &mut Vec, @@ -613,7 +607,7 @@ impl ErrorsAndIncompatibilities { fn unpack_dep( &mut self, - target_label: &ConfiguredTargetLabel, + target_label: &TargetConfiguredTargetLabel, result: anyhow::Result>, check_visibility: CheckVisibility, ) -> Option { @@ -623,7 +617,7 @@ impl ErrorsAndIncompatibilities { } Ok(MaybeCompatible::Incompatible(reason)) => { self.incompats.push(Arc::new(IncompatiblePlatformReason { - target: target_label.dupe(), + target: target_label.inner().dupe(), cause: IncompatiblePlatformReasonCause::Dependency(reason.dupe()), })); } @@ -668,13 +662,13 @@ impl ErrorsAndIncompatibilities { struct GatheredDeps { deps: Vec, exec_deps: SmallMap, - toolchain_deps: SmallSet, + toolchain_deps: SmallSet, plugin_lists: PluginLists, } async fn gather_deps( - target_label: &ConfiguredTargetLabel, - target_node: &TargetNode, + target_label: &TargetConfiguredTargetLabel, + target_node: TargetNodeRef<'_>, attr_cfg_ctx: &(dyn AttrConfigurationContext + Sync), ctx: &DiceComputations, ) -> anyhow::Result<(GatheredDeps, ErrorsAndIncompatibilities)> { @@ -682,7 +676,7 @@ async fn gather_deps( struct Traversal { deps: OrderedMap>, exec_deps: SmallMap, - toolchain_deps: SmallSet, + toolchain_deps: SmallSet, plugin_lists: PluginLists, } @@ -710,7 +704,10 @@ async fn gather_deps( } fn toolchain_dep(&mut self, dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { - self.toolchain_deps.insert(dep.target().dupe()); + self.toolchain_deps + .insert(TargetConfiguredTargetLabel::new_without_exec_cfg( + dep.target().dupe(), + )); Ok(()) } @@ -795,6 +792,8 @@ async fn compute_configured_target_node_no_transition( target_node: TargetNode, ctx: &DiceComputations, ) -> anyhow::Result> { + let partial_target_label = + &TargetConfiguredTargetLabel::new_without_exec_cfg(target_label.dupe()); let target_cfg = target_label.cfg(); let target_cell = target_node.label().pkg().cell_name(); let resolved_configuration = ctx @@ -808,7 +807,7 @@ async fn compute_configured_target_node_no_transition( // Must check for compatibility before evaluating non-compatibility attributes. if let MaybeCompatible::Incompatible(reason) = - check_compatible(target_label, &target_node, &resolved_configuration)? + check_compatible(target_label, target_node.as_ref(), &resolved_configuration)? { return Ok(MaybeCompatible::Incompatible(reason)); } @@ -817,12 +816,12 @@ async fn compute_configured_target_node_no_transition( for (_dep, tr) in target_node.transition_deps() { let resolved_cfg = TRANSITION_CALCULATION .get()? - .apply_transition(ctx, &target_node, target_cfg, tr) + .apply_transition(ctx, target_node.as_ref(), target_cfg, tr) .await?; resolved_transitions.insert(tr.dupe(), resolved_cfg); } - let platform_cfgs = compute_platform_cfgs(ctx, &target_node).await?; + let platform_cfgs = compute_platform_cfgs(ctx, target_node.as_ref()).await?; // We need to collect deps and to ensure that all attrs can be successfully // configured so that we don't need to support propagate configuration errors on attr access. @@ -835,8 +834,13 @@ async fn compute_configured_target_node_no_transition( &resolved_transitions, &platform_cfgs, ); - let (gathered_deps, mut errors_and_incompats) = - gather_deps(target_label, &target_node, &attr_cfg_ctx, ctx).await?; + let (gathered_deps, mut errors_and_incompats) = gather_deps( + partial_target_label, + target_node.as_ref(), + &attr_cfg_ctx, + ctx, + ) + .await?; check_plugin_deps(ctx, target_label, &gathered_deps.plugin_lists).await?; @@ -864,7 +868,7 @@ async fn compute_configured_target_node_no_transition( } else { resolve_execution_platform( ctx, - &target_node, + target_node.as_ref(), &resolved_configuration, &gathered_deps, &attr_cfg_ctx, @@ -906,10 +910,20 @@ async fn compute_configured_target_node_no_transition( let mut exec_deps = Vec::with_capacity(gathered_deps.exec_deps.len()); for dep in toolchain_dep_results { - errors_and_incompats.unpack_dep_into(target_label, dep, CheckVisibility::Yes, &mut deps); + errors_and_incompats.unpack_dep_into( + partial_target_label, + dep, + CheckVisibility::Yes, + &mut deps, + ); } for (dep, check_visibility) in exec_dep_results { - errors_and_incompats.unpack_dep_into(target_label, dep, *check_visibility, &mut exec_deps); + errors_and_incompats.unpack_dep_into( + partial_target_label, + dep, + *check_visibility, + &mut exec_deps, + ); } if let Some(ret) = errors_and_incompats.finalize() { @@ -987,7 +1001,7 @@ async fn compute_configured_target_node( _ => {} } - if let Some(transition_id) = &target_node.0.rule.cfg { + if let Some(transition_id) = &target_node.rule.cfg { #[async_trait] impl Key for ConfiguredTransitionedNodeKey { type Value = buck2_error::Result>; @@ -1013,7 +1027,7 @@ async fn compute_configured_target_node( let cfg = TRANSITION_CALCULATION .get()? - .apply_transition(ctx, &target_node, key.0.cfg(), transition_id) + .apply_transition(ctx, target_node.as_ref(), key.0.cfg(), transition_id) .await?; let configured_target_label = key.0.unconfigured().configure(cfg.single()?.dupe()); diff --git a/app/buck2_configured/src/target.rs b/app/buck2_configured/src/target.rs new file mode 100644 index 0000000000000..6eaa2a0fae024 --- /dev/null +++ b/app/buck2_configured/src/target.rs @@ -0,0 +1,68 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use buck2_core::configuration::data::ConfigurationData; +use buck2_core::target::configured_target_label::ConfiguredTargetLabel; +use buck2_core::target::label::TargetLabel; +use dupe::Dupe; + +/// A wrapper around a configured target label. +/// +/// The semantics of this type are exactly the same as that of a configured target label, except +/// with one distinction - when the target being referred to is a toolchain target, the label will +/// still only be configured with a target platform, not an exec platform. +/// +/// This is used to mark code which deals with configurations of toolchains, but does not actually +/// care about the toolchain's exec platform. +#[derive( + Clone, + Dupe, + Debug, + derive_more::Display, + Hash, + Eq, + PartialEq, + Ord, + PartialOrd, + allocative::Allocative +)] +pub struct TargetConfiguredTargetLabel(ConfiguredTargetLabel); + +impl TargetConfiguredTargetLabel { + pub fn new_without_exec_cfg(label: ConfiguredTargetLabel) -> Self { + Self(label.unconfigured().configure(label.cfg().dupe())) + } + + pub fn new_configure(label: &TargetLabel, cfg: ConfigurationData) -> Self { + Self(label.configure(cfg)) + } + + pub fn unconfigured(&self) -> &TargetLabel { + self.0.unconfigured() + } + + pub fn cfg(&self) -> &ConfigurationData { + self.0.cfg() + } + + pub fn pkg(&self) -> buck2_core::package::PackageLabel { + self.0.pkg() + } + + /// Sets the exec configuration. + /// + /// Should only be used with toolchain targets. + pub fn with_exec_cfg(&self, cfg: ConfigurationData) -> ConfiguredTargetLabel { + self.0.with_exec_cfg(cfg) + } + + pub fn inner(&self) -> &ConfiguredTargetLabel { + &self.0 + } +} diff --git a/app/buck2_core/src/cells/cell_path.rs b/app/buck2_core/src/cells/cell_path.rs index 6bd72c03a5449..8518c986cc139 100644 --- a/app/buck2_core/src/cells/cell_path.rs +++ b/app/buck2_core/src/cells/cell_path.rs @@ -63,18 +63,21 @@ impl CellPath { /// /// ``` /// use buck2_core::cells::cell_path::CellPath; - /// use buck2_core::cells::paths::{CellRelativePathBuf}; /// use buck2_core::cells::name::CellName; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// let path = CellPath::new( /// CellName::testing_new("cell"), - /// CellRelativePathBuf::unchecked_new("foo/bar".into()) + /// CellRelativePathBuf::unchecked_new("foo/bar".into()), /// ); /// let other = ForwardRelativePath::new("baz")?; /// assert_eq!( - /// CellPath::new(CellName::testing_new("cell"), - /// CellRelativePathBuf::unchecked_new("foo/bar/baz".into())), path.join(other) + /// CellPath::new( + /// CellName::testing_new("cell"), + /// CellRelativePathBuf::unchecked_new("foo/bar/baz".into()) + /// ), + /// path.join(other) /// ); /// /// # anyhow::Ok(()) @@ -88,18 +91,20 @@ impl CellPath { /// /// ``` /// use buck2_core::cells::cell_path::CellPath; - /// use buck2_core::cells::paths::{CellRelativePathBuf}; /// use buck2_core::cells::name::CellName; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// /// assert_eq!( - /// Some( - /// CellPath::new(CellName::testing_new("cell"), - /// CellRelativePathBuf::unchecked_new("foo".into())) - /// ), + /// Some(CellPath::new( + /// CellName::testing_new("cell"), + /// CellRelativePathBuf::unchecked_new("foo".into()) + /// )), /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("foo/bar".into()) - /// ).parent().map(|p| p.to_owned()), + /// ) + /// .parent() + /// .map(|p| p.to_owned()), /// ); /// /// # anyhow::Ok(()) @@ -116,15 +121,24 @@ impl CellPath { /// /// ``` /// use buck2_core::cells::cell_path::CellPath; - /// use buck2_core::cells::paths::{CellRelativePathBuf}; /// use buck2_core::cells::name::CellName; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// /// let path = CellPath::testing_new("cell//foo/bar"); /// let mut ancestors = path.ancestors(); /// - /// assert_eq!(ancestors.next(), Some(CellPath::testing_new("cell//foo/bar").as_ref())); - /// assert_eq!(ancestors.next(), Some(CellPath::testing_new("cell//foo").as_ref())); - /// assert_eq!(ancestors.next(), Some(CellPath::testing_new("cell//").as_ref())); + /// assert_eq!( + /// ancestors.next(), + /// Some(CellPath::testing_new("cell//foo/bar").as_ref()) + /// ); + /// assert_eq!( + /// ancestors.next(), + /// Some(CellPath::testing_new("cell//foo").as_ref()) + /// ); + /// assert_eq!( + /// ancestors.next(), + /// Some(CellPath::testing_new("cell//").as_ref()) + /// ); /// assert_eq!(ancestors.next(), None); /// /// # anyhow::Ok(()) @@ -142,13 +156,13 @@ impl CellPath { /// /// ``` /// use buck2_core::cells::cell_path::CellPath; - /// use buck2_core::cells::paths::{CellRelativePathBuf}; /// use buck2_core::cells::name::CellName; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// /// let path = CellPath::new( /// CellName::testing_new("cell"), - /// CellRelativePathBuf::unchecked_new("test/haha/foo.txt".into()) + /// CellRelativePathBuf::unchecked_new("test/haha/foo.txt".into()), /// ); /// /// assert_eq!( @@ -156,7 +170,8 @@ impl CellPath { /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("test".into()), - /// ).as_ref() + /// ) + /// .as_ref() /// )?, /// ForwardRelativePathBuf::unchecked_new("haha/foo.txt".into()) /// ); @@ -165,8 +180,10 @@ impl CellPath { /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("asdf".into()), - /// ).as_ref() - /// ).is_err(), + /// ) + /// .as_ref() + /// ) + /// .is_err(), /// true /// ); /// assert_eq!( @@ -174,8 +191,10 @@ impl CellPath { /// CellPath::new( /// CellName::testing_new("another"), /// CellRelativePathBuf::unchecked_new("test".into()), - /// ).as_ref() - /// ).is_err(), + /// ) + /// .as_ref() + /// ) + /// .is_err(), /// true /// ); /// @@ -193,17 +212,18 @@ impl CellPath { /// normalized. /// /// ``` - /// - /// use buck2_core::cells::paths::CellRelativePathBuf; - /// use buck2_core::cells::name::CellName; /// use std::convert::TryFrom; + /// /// use buck2_core::cells::cell_path::CellPath; + /// use buck2_core::cells::name::CellName; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// /// assert_eq!( /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("foo/bar".into()) - /// ).join_normalized("../baz.txt")?, + /// ) + /// .join_normalized("../baz.txt")?, /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("foo/baz.txt".into()) @@ -214,7 +234,9 @@ impl CellPath { /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("foo".into()) - /// ).join_normalized("../../baz.txt").is_err(), + /// ) + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// @@ -227,20 +249,24 @@ impl CellPath { /// Checks that cell matches and `self` path starts with `base` path /// /// ``` + /// use std::convert::TryFrom; /// - /// use buck2_core::cells::paths::CellRelativePathBuf; /// use buck2_core::cells::cell_path::CellPath; /// use buck2_core::cells::name::CellName; - /// use std::convert::TryFrom; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// /// assert!( /// CellPath::new( /// CellName::testing_new("cell"), /// CellRelativePathBuf::unchecked_new("foo/bar".into()) - /// ).starts_with(CellPath::new( - /// CellName::testing_new("cell"), - /// CellRelativePathBuf::unchecked_new("foo".into()) - /// ).as_ref()), + /// ) + /// .starts_with( + /// CellPath::new( + /// CellName::testing_new("cell"), + /// CellRelativePathBuf::unchecked_new("foo".into()) + /// ) + /// .as_ref() + /// ), /// ); /// /// # anyhow::Ok(()) diff --git a/app/buck2_core/src/cells/mod.rs b/app/buck2_core/src/cells/mod.rs index 842dc8a06753a..8ef6a057491b3 100644 --- a/app/buck2_core/src/cells/mod.rs +++ b/app/buck2_core/src/cells/mod.rs @@ -126,7 +126,6 @@ //! //! # anyhow::Ok(()) //! ``` -//! pub mod alias; pub mod build_file_cell; @@ -173,6 +172,7 @@ use crate::package::PackageLabel; /// Errors from cell creation #[derive(buck2_error::Error, Debug)] +#[buck2(user)] enum CellError { #[error("Cell paths `{1}` and `{2}` had the same alias `{0}`.")] DuplicateAliases(NonEmptyCellAlias, CellRootPathBuf, CellRootPathBuf), @@ -409,13 +409,15 @@ impl CellResolver { /// the 'Package' /// /// ``` - /// use buck2_core::cells::CellResolver; - /// use buck2_core::fs::project_rel_path::{ProjectRelativePath, ProjectRelativePathBuf}; /// use std::convert::TryFrom; + /// /// use buck2_core::cells::cell_path::CellPath; /// use buck2_core::cells::cell_root_path::CellRootPathBuf; /// use buck2_core::cells::name::CellName; /// use buck2_core::cells::paths::CellRelativePathBuf; + /// use buck2_core::cells::CellResolver; + /// use buck2_core::fs::project_rel_path::ProjectRelativePath; + /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// /// let cell_path = ProjectRelativePath::new("my/cell")?; /// let cells = CellResolver::testing_with_name_and_path( @@ -425,7 +427,8 @@ impl CellResolver { /// /// let cell_path = CellPath::new( /// CellName::testing_new("mycell"), - /// CellRelativePathBuf::unchecked_new("some/path".to_owned())); + /// CellRelativePathBuf::unchecked_new("some/path".to_owned()), + /// ); /// /// assert_eq!( /// cells.resolve_path(cell_path.as_ref())?, @@ -442,14 +445,17 @@ impl CellResolver { /// the 'Package' /// /// ``` - /// use buck2_core::cells::CellResolver; - /// use buck2_core::fs::project_rel_path::{ProjectRelativePath, ProjectRelativePathBuf}; - /// use buck2_core::fs::paths::forward_rel_path::{ForwardRelativePathBuf, ForwardRelativePath}; - /// use buck2_core::package::PackageLabel; /// use std::convert::TryFrom; + /// /// use buck2_core::cells::cell_root_path::CellRootPathBuf; /// use buck2_core::cells::name::CellName; /// use buck2_core::cells::paths::CellRelativePath; + /// use buck2_core::cells::CellResolver; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; + /// use buck2_core::fs::project_rel_path::ProjectRelativePath; + /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; + /// use buck2_core::package::PackageLabel; /// /// let cell_path = ProjectRelativePath::new("my/cell")?; /// diff --git a/app/buck2_core/src/cells/paths.rs b/app/buck2_core/src/cells/paths.rs index 2263889c908de..9c6eec00ed5be 100644 --- a/app/buck2_core/src/cells/paths.rs +++ b/app/buck2_core/src/cells/paths.rs @@ -112,9 +112,10 @@ impl CellRelativePath { /// forward, normalized relative path, otherwise error. /// /// ``` - /// use buck2_core::cells::paths::CellRelativePath; /// use std::path::Path; /// + /// use buck2_core::cells::paths::CellRelativePath; + /// /// assert!(CellRelativePath::from_path("foo/bar").is_ok()); /// assert!(CellRelativePath::from_path("").is_ok()); /// assert!(CellRelativePath::from_path("/abs/bar").is_err()); @@ -151,12 +152,17 @@ impl CellRelativePath { /// /// ``` /// use std::path::Path; + /// + /// use buck2_core::cells::paths::CellRelativePath; + /// use buck2_core::cells::paths::CellRelativePathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use buck2_core::cells::paths::{CellRelativePathBuf, CellRelativePath}; /// /// let path = CellRelativePath::from_path("foo/bar")?; /// let other = ForwardRelativePath::new("baz")?; - /// assert_eq!(CellRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), path.join(other)); + /// assert_eq!( + /// CellRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), + /// path.join(other) + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -190,7 +196,10 @@ impl CellRelativePath { /// use buck2_core::cells::paths::CellRelativePath; /// use buck2_core::fs::paths::file_name::FileName; /// - /// assert_eq!(Some(FileName::unchecked_new("bin")), CellRelativePath::from_path("usr/bin")?.file_name()); + /// assert_eq!( + /// Some(FileName::unchecked_new("bin")), + /// CellRelativePath::from_path("usr/bin")?.file_name() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -205,8 +214,8 @@ impl CellRelativePath { /// path is not a 'ForwardRelativePath' /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::cells::paths::CellRelativePath; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// let path = CellRelativePath::from_path("test/haha/foo.txt")?; /// @@ -214,7 +223,11 @@ impl CellRelativePath { /// path.strip_prefix(CellRelativePath::from_path("test")?)?, /// ForwardRelativePath::new("haha/foo.txt")? /// ); - /// assert_eq!(path.strip_prefix(CellRelativePath::from_path("asdf")?).is_err(), true); + /// assert_eq!( + /// path.strip_prefix(CellRelativePath::from_path("asdf")?) + /// .is_err(), + /// true + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -228,7 +241,6 @@ impl CellRelativePath { /// Determines whether `base` is a prefix of `self`. /// /// ``` - /// /// use buck2_core::cells::paths::CellRelativePath; /// /// let path = CellRelativePath::from_path("some/foo")?; @@ -246,6 +258,7 @@ impl CellRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::cells::paths::CellRelativePath; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// @@ -285,10 +298,12 @@ impl CellRelativePath { /// Extracts the extension of [`self.file_name`], if possible. /// /// ``` - /// /// use buck2_core::cells::paths::CellRelativePath; /// - /// assert_eq!(Some("rs"), CellRelativePath::from_path("hi/foo.rs")?.extension()); + /// assert_eq!( + /// Some("rs"), + /// CellRelativePath::from_path("hi/foo.rs")?.extension() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -300,17 +315,20 @@ impl CellRelativePath { /// normalized. /// /// ``` - /// - /// use buck2_core::cells::paths::{CellRelativePath, CellRelativePathBuf}; /// use std::convert::TryFrom; /// + /// use buck2_core::cells::paths::CellRelativePath; + /// use buck2_core::cells::paths::CellRelativePathBuf; + /// /// assert_eq!( /// CellRelativePath::from_path("foo/bar")?.join_normalized("../baz.txt")?, /// CellRelativePathBuf::unchecked_new("foo/baz.txt".into()), /// ); /// /// assert_eq!( - /// CellRelativePath::from_path("foo")?.join_normalized("../../baz.txt").is_err(), + /// CellRelativePath::from_path("foo")? + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// @@ -334,22 +352,10 @@ impl CellRelativePath { /// let p = CellRelativePath::from_path("foo/bar/baz")?; /// let mut it = p.iter(); /// - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("foo")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("bar")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("baz")) - /// ); - /// assert_eq!( - /// it.next(), - /// None - /// ); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("foo"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("bar"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("baz"))); + /// assert_eq!(it.next(), None); /// /// # anyhow::Ok(()) /// ``` @@ -369,14 +375,17 @@ impl CellRelativePath { impl<'a> From<&'a ForwardRelativePath> for &'a CellRelativePath { /// /// ``` + /// use std::convert::From; /// /// use buck2_core::cells::paths::CellRelativePath; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use std::convert::From; /// /// let f = ForwardRelativePath::new("foo")?; /// - /// assert_eq!(<&CellRelativePath>::from(f), CellRelativePath::from_path("foo")?); + /// assert_eq!( + /// <&CellRelativePath>::from(f), + /// CellRelativePath::from_path("foo")? + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -453,9 +462,9 @@ impl<'a> TryFrom<&'a str> for &'a CellRelativePath { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::cells::paths::CellRelativePath; - /// use std::convert::TryFrom; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// assert!(<&CellRelativePath>::try_from("foo/bar").is_ok()); @@ -475,9 +484,9 @@ impl<'a> TryFrom<&'a RelativePath> for &'a CellRelativePath { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::cells::paths::CellRelativePath; - /// use std::convert::TryFrom; /// use buck2_core::fs::paths::RelativePath; /// /// assert!(<&CellRelativePath>::try_from(RelativePath::new("foo/bar")).is_ok()); @@ -498,9 +507,9 @@ impl TryFrom for CellRelativePathBuf { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::cells::paths::CellRelativePathBuf; - /// use std::convert::TryFrom; /// /// assert!(CellRelativePathBuf::try_from("foo/bar".to_owned()).is_ok()); /// assert!(CellRelativePathBuf::try_from("".to_owned()).is_ok()); @@ -522,9 +531,10 @@ impl TryFrom for CellRelativePathBuf { /// conversion) /// /// ``` + /// use std::convert::TryFrom; + /// /// use buck2_core::cells::paths::CellRelativePathBuf; /// use buck2_core::fs::paths::RelativePathBuf; - /// use std::convert::TryFrom; /// /// assert!(CellRelativePathBuf::try_from(RelativePathBuf::from("foo/bar")).is_ok()); /// assert!(CellRelativePathBuf::try_from(RelativePathBuf::from("")).is_ok()); @@ -544,11 +554,11 @@ impl TryFrom for CellRelativePathBuf { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::cells::paths::CellRelativePathBuf; /// use std::convert::TryFrom; /// use std::path::PathBuf; /// + /// use buck2_core::cells::paths::CellRelativePathBuf; + /// /// assert!(CellRelativePathBuf::try_from(PathBuf::from("foo/bar")).is_ok()); /// assert!(CellRelativePathBuf::try_from(PathBuf::from("")).is_ok()); /// assert!(CellRelativePathBuf::try_from(PathBuf::from("/abs/bar")).is_err()); diff --git a/app/buck2_core/src/configuration/data.rs b/app/buck2_core/src/configuration/data.rs index 30de01919d5d1..ee1e89956d30a 100644 --- a/app/buck2_core/src/configuration/data.rs +++ b/app/buck2_core/src/configuration/data.rs @@ -30,6 +30,7 @@ use crate::configuration::constraints::ConstraintValue; use crate::configuration::hash::ConfigurationHash; #[derive(Debug, buck2_error::Error)] +#[buck2(user)] enum ConfigurationError { #[error( "Attempted to access the configuration data for the {0} platform. \ diff --git a/app/buck2_core/src/configuration/mod.rs b/app/buck2_core/src/configuration/mod.rs index 32cd85597be21..37698d39d1548 100644 --- a/app/buck2_core/src/configuration/mod.rs +++ b/app/buck2_core/src/configuration/mod.rs @@ -16,7 +16,6 @@ //! under a "transition". Multiple distinct configurations may be applied to the //! transitive graph, effectively duplicating the graph to create two distinct //! graphs with different build behaviours (split-transitions). -//! pub mod bound_id; pub mod bound_label; diff --git a/app/buck2_core/src/fs/buck_out_path.rs b/app/buck2_core/src/fs/buck_out_path.rs index 51c0906d760a1..4ec7277994ad6 100644 --- a/app/buck2_core/src/fs/buck_out_path.rs +++ b/app/buck2_core/src/fs/buck_out_path.rs @@ -75,6 +75,10 @@ impl BuckOutPath { pub fn path(&self) -> &ForwardRelativePath { &self.0.path } + + pub fn len(&self) -> usize { + self.0.path.as_str().len() + } } #[derive(Clone, Debug, Display, Eq, PartialEq)] diff --git a/app/buck2_core/src/fs/paths/abs_norm_path.rs b/app/buck2_core/src/fs/paths/abs_norm_path.rs index 481f3ff6c1c17..eb46b098b6ab9 100644 --- a/app/buck2_core/src/fs/paths/abs_norm_path.rs +++ b/app/buck2_core/src/fs/paths/abs_norm_path.rs @@ -108,9 +108,9 @@ impl AbsNormPath { /// /// assert!(AbsNormPath::new("foo/bar").is_err()); /// if cfg!(windows) { - /// assert!(AbsNormPath::new("C:\\foo\\bar").is_ok()); + /// assert!(AbsNormPath::new("C:\\foo\\bar").is_ok()); /// } else { - /// assert!(AbsNormPath::new("/foo/bar").is_ok()); + /// assert!(AbsNormPath::new("/foo/bar").is_ok()); /// } /// ``` pub fn new>(p: &P) -> anyhow::Result<&AbsNormPath> { @@ -123,15 +123,25 @@ impl AbsNormPath { /// /// ``` /// use std::path::Path; - /// use buck2_core::fs::paths::abs_norm_path::{AbsNormPath, AbsNormPathBuf}; + /// + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// if cfg!(not(windows)) { /// let abs_path = AbsNormPath::new("/my")?; - /// assert_eq!(AbsNormPathBuf::from("/my/foo/bar".into())?, abs_path.join(ForwardRelativePath::new("foo/bar")?)); + /// assert_eq!( + /// AbsNormPathBuf::from("/my/foo/bar".into())?, + /// abs_path.join(ForwardRelativePath::new("foo/bar")?) + /// ); /// } else { /// let abs_path = AbsNormPath::new("C:\\my")?; - /// assert_eq!("C:\\my\\foo\\bar", abs_path.join(ForwardRelativePath::new("foo/bar")?).to_string()); + /// assert_eq!( + /// "C:\\my\\foo\\bar", + /// abs_path + /// .join(ForwardRelativePath::new("foo/bar")?) + /// .to_string() + /// ); /// } /// # anyhow::Ok(()) /// ``` @@ -153,6 +163,7 @@ impl AbsNormPath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; /// /// if cfg!(not(windows)) { @@ -160,19 +171,13 @@ impl AbsNormPath { /// Some(AbsNormPath::new("/")?), /// AbsNormPath::new("/my")?.parent() /// ); - /// assert_eq!( - /// None, - /// AbsNormPath::new("/")?.parent() - /// ); + /// assert_eq!(None, AbsNormPath::new("/")?.parent()); /// } else { /// assert_eq!( /// Some(AbsNormPath::new("c:/")?), /// AbsNormPath::new("c:/my")?.parent() /// ); - /// assert_eq!( - /// None, - /// AbsNormPath::new("c:/")?.parent() - /// ); + /// assert_eq!(None, AbsNormPath::new("c:/")?.parent()); /// } /// /// # anyhow::Ok(()) @@ -188,7 +193,9 @@ impl AbsNormPath { /// path is not a 'ForwardRelativePath' /// /// ``` - /// use std::{borrow::Cow, path::Path}; + /// use std::borrow::Cow; + /// use std::path::Path; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// @@ -232,8 +239,16 @@ impl AbsNormPath { /// shared_path.strip_prefix(AbsNormPath::new(r"\\?\UNC\server\share\foo")?)?, /// Cow::Borrowed(ForwardRelativePath::new("bar.txt")?) /// ); - /// assert!(shared_path.strip_prefix(AbsNormPath::new(r"\\server\share2\foo")?).is_err()); - /// assert!(shared_path.strip_prefix(AbsNormPath::new(r"\\server\share\fo")?).is_err()); + /// assert!( + /// shared_path + /// .strip_prefix(AbsNormPath::new(r"\\server\share2\foo")?) + /// .is_err() + /// ); + /// assert!( + /// shared_path + /// .strip_prefix(AbsNormPath::new(r"\\server\share\fo")?) + /// .is_err() + /// ); /// } /// /// # anyhow::Ok(()) @@ -266,6 +281,7 @@ impl AbsNormPath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; /// /// if cfg!(not(windows)) { @@ -318,6 +334,7 @@ impl AbsNormPath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; /// /// if cfg!(not(windows)) { @@ -337,7 +354,8 @@ impl AbsNormPath { /// Build an owned `AbsPathBuf`, joined with the given path and normalized. /// /// ``` - /// use buck2_core::fs::paths::abs_norm_path::{AbsNormPath, AbsNormPathBuf}; + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// /// if cfg!(not(windows)) { /// assert_eq!( @@ -346,7 +364,9 @@ impl AbsNormPath { /// ); /// /// assert_eq!( - /// AbsNormPath::new("/foo")?.join_normalized("../../baz.txt").is_err(), + /// AbsNormPath::new("/foo")? + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// } else { @@ -356,7 +376,9 @@ impl AbsNormPath { /// ); /// /// assert_eq!( - /// AbsNormPath::new("c:/foo")?.join_normalized("../../baz.txt").is_err(), + /// AbsNormPath::new("c:/foo")? + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// } @@ -411,11 +433,23 @@ impl AbsNormPath { /// assert_eq!("D", AbsNormPath::new("d:/foo/bar")?.windows_prefix()?); /// assert_eq!("D", AbsNormPath::new(r"D:\foo\bar")?.windows_prefix()?); /// assert_eq!("E", AbsNormPath::new(r"\\?\E:\foo\bar")?.windows_prefix()?); - /// assert_eq!("server\\share", AbsNormPath::new(r"\\server\share")?.windows_prefix()?); - /// assert_eq!("server\\share", AbsNormPath::new(r"\\server\share\foo\bar")?.windows_prefix()?); - /// assert_eq!("server\\share", AbsNormPath::new(r"\\?\UNC\server\share")?.windows_prefix()?); + /// assert_eq!( + /// "server\\share", + /// AbsNormPath::new(r"\\server\share")?.windows_prefix()? + /// ); + /// assert_eq!( + /// "server\\share", + /// AbsNormPath::new(r"\\server\share\foo\bar")?.windows_prefix()? + /// ); + /// assert_eq!( + /// "server\\share", + /// AbsNormPath::new(r"\\?\UNC\server\share")?.windows_prefix()? + /// ); /// assert_eq!("COM42", AbsNormPath::new(r"\\.\COM42")?.windows_prefix()?); - /// assert_eq!("COM42", AbsNormPath::new(r"\\.\COM42\foo\bar")?.windows_prefix()?); + /// assert_eq!( + /// "COM42", + /// AbsNormPath::new(r"\\.\COM42\foo\bar")?.windows_prefix()? + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -451,16 +485,41 @@ impl AbsNormPath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; /// - /// assert_eq!(Path::new(""), AbsNormPath::new("C:/")?.strip_windows_prefix()?); - /// assert_eq!(Path::new(""), AbsNormPath::new("C:\\")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("foo/bar"), AbsNormPath::new("d:/foo/bar")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("foo\\bar"), AbsNormPath::new(r"D:\foo\bar")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("foo\\bar"), AbsNormPath::new(r"\\?\D:\foo\bar")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("path"), AbsNormPath::new(r"\\server\share\path")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("path"), AbsNormPath::new(r"\\?\UNC\server\share\path")?.strip_windows_prefix()?); - /// assert_eq!(Path::new("abc"), AbsNormPath::new(r"\\.\COM42\abc")?.strip_windows_prefix()?); + /// assert_eq!( + /// Path::new(""), + /// AbsNormPath::new("C:/")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new(""), + /// AbsNormPath::new("C:\\")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("foo/bar"), + /// AbsNormPath::new("d:/foo/bar")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("foo\\bar"), + /// AbsNormPath::new(r"D:\foo\bar")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("foo\\bar"), + /// AbsNormPath::new(r"\\?\D:\foo\bar")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("path"), + /// AbsNormPath::new(r"\\server\share\path")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("path"), + /// AbsNormPath::new(r"\\?\UNC\server\share\path")?.strip_windows_prefix()? + /// ); + /// assert_eq!( + /// Path::new("abc"), + /// AbsNormPath::new(r"\\.\COM42\abc")?.strip_windows_prefix()? + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -539,24 +598,26 @@ impl AbsNormPathBuf { /// Pushes a `ForwardRelativePath` to the existing buffer /// ``` - /// /// use std::path::PathBuf; + /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// - /// let prefix = if cfg!(windows) { - /// "C:" - /// } else { - /// "" - /// }; + /// let prefix = if cfg!(windows) { "C:" } else { "" }; /// /// let mut path = AbsNormPathBuf::try_from(format!("{prefix}/foo")).unwrap(); /// path.push(ForwardRelativePath::unchecked_new("bar")); /// - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar")).unwrap(), + /// path + /// ); /// /// path.push(ForwardRelativePath::unchecked_new("more/file.rs")); - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/file.rs")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/file.rs")).unwrap(), + /// path + /// ); /// ``` pub fn push>(&mut self, path: P) { if cfg!(windows) { @@ -570,35 +631,48 @@ impl AbsNormPathBuf { /// Note that this does not visit the filesystem to resolve `..`s. Instead, it cancels out the /// components directly, similar to `join_normalized`. /// ``` - /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::paths::RelativePath; /// - /// let prefix = if cfg!(windows) { - /// "C:" - /// } else { - /// "" - /// }; + /// let prefix = if cfg!(windows) { "C:" } else { "" }; /// /// let mut path = AbsNormPathBuf::try_from(format!("{prefix}/foo")).unwrap(); /// path.push_normalized(RelativePath::new("bar"))?; /// - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar")).unwrap(), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("more/file.rs"))?; - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/file.rs")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/file.rs")).unwrap(), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("../other.rs"))?; - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/other.rs")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more/other.rs")).unwrap(), + /// path + /// ); /// /// path.push_normalized(RelativePath::new(".."))?; - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo/bar/more")).unwrap(), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("../.."))?; - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/foo")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/foo")).unwrap(), + /// path + /// ); /// /// path.push_normalized(RelativePath::new(".."))?; - /// assert_eq!(AbsNormPathBuf::try_from(format!("{prefix}/")).unwrap(), path); + /// assert_eq!( + /// AbsNormPathBuf::try_from(format!("{prefix}/")).unwrap(), + /// path + /// ); /// /// assert!(path.push_normalized(RelativePath::new("..")).is_err()); /// @@ -636,9 +710,9 @@ impl TryFrom for AbsNormPathBuf { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; - /// use std::convert::TryFrom; /// /// assert!(AbsNormPathBuf::try_from("relative/bar".to_owned()).is_err()); /// @@ -674,11 +748,11 @@ impl TryFrom for AbsNormPathBuf { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use std::convert::TryFrom; /// use std::path::PathBuf; /// + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; + /// /// assert!(AbsNormPathBuf::try_from(PathBuf::from("relative/bar")).is_err()); /// /// if cfg!(not(windows)) { diff --git a/app/buck2_core/src/fs/paths/cmp_impls.rs b/app/buck2_core/src/fs/paths/cmp_impls.rs index bb6bf86ad61d1..14ebd2e3155bd 100644 --- a/app/buck2_core/src/fs/paths/cmp_impls.rs +++ b/app/buck2_core/src/fs/paths/cmp_impls.rs @@ -9,7 +9,6 @@ //! //! General macros useful for path declaration -//! use std::cmp; diff --git a/app/buck2_core/src/fs/paths/file_name.rs b/app/buck2_core/src/fs/paths/file_name.rs index f8c6bffd3c5df..aa1e11784f9b8 100644 --- a/app/buck2_core/src/fs/paths/file_name.rs +++ b/app/buck2_core/src/fs/paths/file_name.rs @@ -177,7 +177,6 @@ impl FileName { /// Extracts the extension of [`self.file_name`], if possible. /// /// ``` - /// /// use buck2_core::fs::paths::file_name::FileName; /// /// assert_eq!(Some("rs"), FileName::new("foo.rs")?.extension()); diff --git a/app/buck2_core/src/fs/paths/forward_rel_path.rs b/app/buck2_core/src/fs/paths/forward_rel_path.rs index 5bec1ed3a66b0..359119179cdc4 100644 --- a/app/buck2_core/src/fs/paths/forward_rel_path.rs +++ b/app/buck2_core/src/fs/paths/forward_rel_path.rs @@ -120,9 +120,10 @@ impl ForwardRelativePath { /// normalized relative path, otherwise error. /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use std::path::Path; /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// /// assert!(ForwardRelativePath::new("foo/bar").is_ok()); /// assert!(ForwardRelativePath::new("").is_ok()); /// assert!(ForwardRelativePath::new("./bar").is_err()); @@ -177,9 +178,10 @@ impl ForwardRelativePath { /// path based on the supplied root. /// /// ``` - /// /// use std::path::Path; - /// use buck2_core::fs::paths::abs_norm_path::{AbsNormPath, AbsNormPathBuf}; + /// + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// if cfg!(not(windows)) { @@ -216,11 +218,16 @@ impl ForwardRelativePath { /// /// ``` /// use std::path::Path; - /// use buck2_core::fs::paths::forward_rel_path::{ForwardRelativePathBuf, ForwardRelativePath}; + /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// /// let path = ForwardRelativePath::new("foo/bar")?; /// let other = ForwardRelativePath::new("baz")?; - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), path.join(other)); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), + /// path.join(other) + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -253,10 +260,7 @@ impl ForwardRelativePath { /// Some(ForwardRelativePath::new("")?), /// ForwardRelativePath::new("foo")?.parent() /// ); - /// assert_eq!( - /// None, - /// ForwardRelativePath::new("")?.parent() - /// ); + /// assert_eq!(None, ForwardRelativePath::new("")?.parent()); /// /// # anyhow::Ok(()) /// ``` @@ -281,12 +285,21 @@ impl ForwardRelativePath { /// a directory, this is the directory name. /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::fs::paths::file_name::FileName; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// - /// assert_eq!(Some(FileName::unchecked_new("ls")), ForwardRelativePath::new("usr/bin/ls")?.file_name()); - /// assert_eq!(Some(FileName::unchecked_new("bin")), ForwardRelativePath::new("usr/bin")?.file_name()); - /// assert_eq!(Some(FileName::unchecked_new("usr")), ForwardRelativePath::new("usr")?.file_name()); + /// assert_eq!( + /// Some(FileName::unchecked_new("ls")), + /// ForwardRelativePath::new("usr/bin/ls")?.file_name() + /// ); + /// assert_eq!( + /// Some(FileName::unchecked_new("bin")), + /// ForwardRelativePath::new("usr/bin")?.file_name() + /// ); + /// assert_eq!( + /// Some(FileName::unchecked_new("usr")), + /// ForwardRelativePath::new("usr")?.file_name() + /// ); /// assert_eq!(None, ForwardRelativePath::new("")?.file_name()); /// /// # anyhow::Ok(()) @@ -351,7 +364,11 @@ impl ForwardRelativePath { /// path.strip_prefix(ForwardRelativePath::new("")?)?, /// ForwardRelativePath::new("test/haha/foo.txt")? /// ); - /// assert_eq!(path.strip_prefix(ForwardRelativePath::new("asdf")?).is_err(), true); + /// assert_eq!( + /// path.strip_prefix(ForwardRelativePath::new("asdf")?) + /// .is_err(), + /// true + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -387,7 +404,6 @@ impl ForwardRelativePath { /// Determines whether `base` is a prefix of `self`. /// /// ``` - /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// let path = ForwardRelativePath::new("some/foo")?; @@ -410,6 +426,7 @@ impl ForwardRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// /// let path = ForwardRelativePath::new("some/foo")?; @@ -445,7 +462,10 @@ impl ForwardRelativePath { /// let path = ForwardRelativePath::new("foo.rs")?; /// /// assert_eq!(Some("foo"), path.file_stem()); - /// assert_eq!(Some("foo.bar"), ForwardRelativePath::new("hi/foo.bar.rs")?.file_stem()); + /// assert_eq!( + /// Some("foo.bar"), + /// ForwardRelativePath::new("hi/foo.bar.rs")?.file_stem() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -466,11 +486,16 @@ impl ForwardRelativePath { /// Extracts the extension of [`self.file_name`], if possible. /// /// ``` - /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// - /// assert_eq!(Some("rs"), ForwardRelativePath::new("hi/foo.rs")?.extension()); - /// assert_eq!(Some("rs"), ForwardRelativePath::new("hi/foo.bar.rs")?.extension()); + /// assert_eq!( + /// Some("rs"), + /// ForwardRelativePath::new("hi/foo.rs")?.extension() + /// ); + /// assert_eq!( + /// Some("rs"), + /// ForwardRelativePath::new("hi/foo.bar.rs")?.extension() + /// ); /// assert_eq!(None, ForwardRelativePath::new(".git")?.extension()); /// assert_eq!(None, ForwardRelativePath::new("foo/.git")?.extension()); /// assert_eq!(None, ForwardRelativePath::new("")?.extension()); @@ -502,8 +527,8 @@ impl ForwardRelativePath { /// normalized. /// /// ``` - /// - /// use buck2_core::fs::paths::forward_rel_path::{ForwardRelativePath, ForwardRelativePathBuf}; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// /// assert_eq!( /// ForwardRelativePathBuf::unchecked_new("foo/baz.txt".into()), @@ -511,7 +536,9 @@ impl ForwardRelativePath { /// ); /// /// assert_eq!( - /// ForwardRelativePath::new("foo")?.join_normalized("../../baz.txt").is_err(), + /// ForwardRelativePath::new("foo")? + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// @@ -543,26 +570,11 @@ impl ForwardRelativePath { /// let p = ForwardRelativePath::new("foo/bar/baz")?; /// let mut it = p.iter(); /// - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("foo")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("bar")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("baz")) - /// ); - /// assert_eq!( - /// it.next(), - /// None - /// ); - /// assert_eq!( - /// it.next(), - /// None - /// ); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("foo"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("bar"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("baz"))); + /// assert_eq!(it.next(), None); + /// assert_eq!(it.next(), None); /// /// # anyhow::Ok(()) /// ``` @@ -594,10 +606,7 @@ impl ForwardRelativePath { /// p.strip_prefix_components(3), /// Some(ForwardRelativePath::new("")?), /// ); - /// assert_eq!( - /// p.strip_prefix_components(4), - /// None, - /// ); + /// assert_eq!(p.strip_prefix_components(4), None,); /// # anyhow::Ok(()) /// ``` pub fn strip_prefix_components(&self, components: usize) -> Option<&Self> { @@ -675,22 +684,35 @@ impl ForwardRelativePathBuf { /// Pushes a `ForwardRelativePath` to the existing buffer /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::{ForwardRelativePath, ForwardRelativePathBuf}; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// /// let mut path = ForwardRelativePathBuf::unchecked_new("foo".to_owned()); /// path.push(ForwardRelativePath::unchecked_new("bar")); /// - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar".to_owned()), + /// path + /// ); /// /// path.push(ForwardRelativePath::unchecked_new("more/file.rs")); - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), + /// path + /// ); /// /// path.push(ForwardRelativePath::empty()); - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), + /// path + /// ); /// /// let mut path = ForwardRelativePathBuf::unchecked_new("".to_owned()); /// path.push(ForwardRelativePath::unchecked_new("foo")); - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo".to_owned()), + /// path + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -732,26 +754,40 @@ impl ForwardRelativePathBuf { /// components directly, similar to `join_normalized`. /// /// ``` - /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// use buck2_core::fs::paths::RelativePath; /// /// let mut path = ForwardRelativePathBuf::unchecked_new("foo".to_owned()); /// path.push_normalized(RelativePath::new("bar"))?; /// - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar".to_owned()), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("more/file.rs"))?; - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/more/file.rs".to_owned()), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("../other.rs"))?; - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/more/other.rs".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/more/other.rs".to_owned()), + /// path + /// ); /// /// path.push_normalized(RelativePath::new(".."))?; - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo/bar/more".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo/bar/more".to_owned()), + /// path + /// ); /// /// path.push_normalized(RelativePath::new("../.."))?; - /// assert_eq!(ForwardRelativePathBuf::unchecked_new("foo".to_owned()), path); + /// assert_eq!( + /// ForwardRelativePathBuf::unchecked_new("foo".to_owned()), + /// path + /// ); /// /// path.push_normalized(RelativePath::new(".."))?; /// assert_eq!(ForwardRelativePathBuf::unchecked_new("".to_owned()), path); @@ -895,9 +931,9 @@ impl<'a> TryFrom<&'a str> for &'a ForwardRelativePath { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use std::convert::TryFrom; /// /// assert!(<&ForwardRelativePath>::try_from("foo/bar").is_ok()); /// assert!(<&ForwardRelativePath>::try_from("").is_ok()); @@ -928,11 +964,11 @@ impl<'a> TryFrom<&'a Path> for &'a ForwardRelativePath { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use std::convert::TryFrom; /// use std::path::Path; /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// /// assert!(<&ForwardRelativePath>::try_from(Path::new("foo/bar")).is_ok()); /// assert!(<&ForwardRelativePath>::try_from(Path::new("")).is_ok()); /// assert!(<&ForwardRelativePath>::try_from(Path::new("./bar")).is_err()); @@ -958,9 +994,9 @@ impl<'a> TryFrom<&'a RelativePath> for &'a ForwardRelativePath { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use std::convert::TryFrom; /// use buck2_core::fs::paths::RelativePath; /// /// assert!(<&ForwardRelativePath>::try_from(RelativePath::new("foo/bar")).is_ok()); @@ -990,9 +1026,9 @@ impl TryFrom for ForwardRelativePathBuf { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; - /// use std::convert::TryFrom; /// /// assert!(ForwardRelativePathBuf::try_from("foo/bar".to_owned()).is_ok()); /// assert!(ForwardRelativePathBuf::try_from("".to_owned()).is_ok()); @@ -1016,11 +1052,12 @@ impl TryFrom for ForwardRelativePathBuf { /// no allocation conversion /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; - /// use buck2_core::fs::paths::RelativePathBuf; /// use std::convert::TryFrom; /// use std::path::PathBuf; /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; + /// use buck2_core::fs::paths::RelativePathBuf; + /// /// assert!(ForwardRelativePathBuf::try_from(PathBuf::from("foo/bar")).is_ok()); /// assert!(ForwardRelativePathBuf::try_from(PathBuf::from("")).is_ok()); /// assert!(ForwardRelativePathBuf::try_from(PathBuf::from("./bar")).is_err()); @@ -1045,9 +1082,10 @@ impl TryFrom for ForwardRelativePathBuf { /// no allocation conversion /// /// ``` + /// use std::convert::TryFrom; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; /// use buck2_core::fs::paths::RelativePathBuf; - /// use std::convert::TryFrom; /// /// assert!(ForwardRelativePathBuf::try_from(RelativePathBuf::from("foo/bar")).is_ok()); /// assert!(ForwardRelativePathBuf::try_from(RelativePathBuf::from("")).is_ok()); diff --git a/app/buck2_core/src/fs/paths/mod.rs b/app/buck2_core/src/fs/paths/mod.rs index e3db0d6a54fa7..a0536e466df8b 100644 --- a/app/buck2_core/src/fs/paths/mod.rs +++ b/app/buck2_core/src/fs/paths/mod.rs @@ -21,7 +21,6 @@ //! 'AbsPath' are absolute paths, meaning they must start with a directory root //! of either `/` or some windows root directory like `c:`. These behave //! roughly like 'Path'. -//! pub mod abs_norm_path; pub mod abs_path; diff --git a/app/buck2_core/src/fs/project.rs b/app/buck2_core/src/fs/project.rs index 422483b1b360f..003dd69ad3766 100644 --- a/app/buck2_core/src/fs/project.rs +++ b/app/buck2_core/src/fs/project.rs @@ -102,9 +102,9 @@ impl ProjectRoot { /// `project root`, yielding a 'AbsPathBuf' /// /// ``` + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::project::ProjectRoot; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; - /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// /// if cfg!(not(windows)) { /// let root = AbsNormPathBuf::from("/usr/local/fbsource/".into())?; @@ -134,9 +134,10 @@ impl ProjectRoot { /// Takes a 'ProjectRelativePath' and converts it to a 'Path' that is relative to the project root. /// /// ``` - /// use buck2_core::fs::project::{ProjectRoot}; - /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use std::path::PathBuf; + /// + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; + /// use buck2_core::fs::project::ProjectRoot; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// let root = if cfg!(not(windows)) { @@ -166,9 +167,11 @@ impl ProjectRoot { /// /// ``` /// use std::borrow::Cow; - /// use buck2_core::fs::project_rel_path::ProjectRelativePath; - /// use buck2_core::fs::paths::abs_norm_path::{AbsNormPathBuf, AbsNormPath}; + /// + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPath; + /// use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; /// use buck2_core::fs::project::ProjectRoot; + /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// if cfg!(not(windows)) { /// let root = AbsNormPathBuf::from("/usr/local/fbsource/".into())?; diff --git a/app/buck2_core/src/fs/project_rel_path.rs b/app/buck2_core/src/fs/project_rel_path.rs index a73e7f8b49ae9..1e217bf48fee0 100644 --- a/app/buck2_core/src/fs/project_rel_path.rs +++ b/app/buck2_core/src/fs/project_rel_path.rs @@ -22,12 +22,16 @@ //! //! Sample uses //! ``` -//! use buck2_core::fs::project::ProjectRoot; -//! use buck2_core::fs::project_rel_path::{ProjectRelativePathBuf, ProjectRelativePath}; -//! use buck2_core::fs::paths::abs_norm_path::{AbsNormPathBuf, AbsNormPath}; +//! use std::borrow::Cow; +//! use std::convert::TryFrom; +//! +//! use buck2_core::fs::paths::abs_norm_path::AbsNormPath; +//! use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; //! use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; +//! use buck2_core::fs::project::ProjectRoot; +//! use buck2_core::fs::project_rel_path::ProjectRelativePath; +//! use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; //! use relative_path::RelativePath; -//! use std::{borrow::Cow, convert::TryFrom}; //! //! let root = if cfg!(not(windows)) { //! AbsNormPathBuf::from("/usr/local/fbsource/".into())? @@ -43,18 +47,26 @@ //! let fs = ProjectRoot::new_unchecked(root); //! let project_rel = fs.relativize(some_path)?; //! -//! assert_eq!(Cow::Borrowed(ProjectRelativePath::new("buck/BUCK")?), project_rel); +//! assert_eq!( +//! Cow::Borrowed(ProjectRelativePath::new("buck/BUCK")?), +//! project_rel +//! ); //! assert_eq!(some_path.to_buf(), fs.resolve(project_rel.as_ref())); //! //! let rel_path = RelativePath::new("../src"); //! let project_rel_2 = project_rel.join_normalized(rel_path)?; -//! assert_eq!(ProjectRelativePathBuf::try_from("buck/src".to_owned())?, project_rel_2); +//! assert_eq!( +//! ProjectRelativePathBuf::try_from("buck/src".to_owned())?, +//! project_rel_2 +//! ); //! -//! assert_eq!(some_path.join_normalized(rel_path)?, fs.resolve(&project_rel_2).to_buf()); +//! assert_eq!( +//! some_path.join_normalized(rel_path)?, +//! fs.resolve(&project_rel_2).to_buf() +//! ); //! //! # anyhow::Ok(()) //! ``` -//! use std::borrow::Borrow; use std::ops::Deref; @@ -156,6 +168,7 @@ impl ProjectRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// assert!(ProjectRelativePath::new("foo/bar").is_ok()); @@ -191,12 +204,17 @@ impl ProjectRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use buck2_core::fs::project_rel_path::{ProjectRelativePathBuf, ProjectRelativePath}; + /// use buck2_core::fs::project_rel_path::ProjectRelativePath; + /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// /// let path = ProjectRelativePath::new("foo/bar")?; /// let other = ForwardRelativePath::new("baz")?; - /// assert_eq!(ProjectRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), path.join(other)); + /// assert_eq!( + /// ProjectRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), + /// path.join(other) + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -230,7 +248,10 @@ impl ProjectRelativePath { /// use buck2_core::fs::paths::file_name::FileName; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// - /// assert_eq!(Some(FileName::unchecked_new("bin")), ProjectRelativePath::new("usr/bin")?.file_name()); + /// assert_eq!( + /// Some(FileName::unchecked_new("bin")), + /// ProjectRelativePath::new("usr/bin")?.file_name() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -246,7 +267,6 @@ impl ProjectRelativePath { /// /// ``` /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// let path = ProjectRelativePath::new("test/haha/foo.txt")?; @@ -255,7 +275,11 @@ impl ProjectRelativePath { /// path.strip_prefix(ProjectRelativePath::new("test")?)?, /// ForwardRelativePath::new("haha/foo.txt")? /// ); - /// assert_eq!(path.strip_prefix(ProjectRelativePath::new("asdf")?).is_err(), true); + /// assert_eq!( + /// path.strip_prefix(ProjectRelativePath::new("asdf")?) + /// .is_err(), + /// true + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -296,8 +320,8 @@ impl ProjectRelativePath { /// /// ``` /// use std::path::Path; - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// let path = ProjectRelativePath::new("some/foo")?; @@ -338,7 +362,10 @@ impl ProjectRelativePath { /// ``` /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// - /// assert_eq!(Some("rs"), ProjectRelativePath::new("hi/foo.rs")?.extension()); + /// assert_eq!( + /// Some("rs"), + /// ProjectRelativePath::new("hi/foo.rs")?.extension() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -351,7 +378,9 @@ impl ProjectRelativePath { /// /// ``` /// use std::convert::TryFrom; - /// use buck2_core::fs::project_rel_path::{ProjectRelativePath, ProjectRelativePathBuf}; + /// + /// use buck2_core::fs::project_rel_path::ProjectRelativePath; + /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// /// assert_eq!( /// ProjectRelativePath::new("foo/bar")?.join_normalized("../baz.txt")?, @@ -359,7 +388,9 @@ impl ProjectRelativePath { /// ); /// /// assert_eq!( - /// ProjectRelativePath::new("foo")?.join_normalized("../../baz.txt").is_err(), + /// ProjectRelativePath::new("foo")? + /// .join_normalized("../../baz.txt") + /// .is_err(), /// true /// ); /// @@ -383,22 +414,10 @@ impl ProjectRelativePath { /// let p = ProjectRelativePath::new("foo/bar/baz")?; /// let mut it = p.iter(); /// - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("foo")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("bar")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("baz")) - /// ); - /// assert_eq!( - /// it.next(), - /// None - /// ); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("foo"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("bar"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("baz"))); + /// assert_eq!(it.next(), None); /// /// # anyhow::Ok(()) /// ``` @@ -414,13 +433,17 @@ impl ProjectRelativePath { impl<'a> From<&'a ForwardRelativePath> for &'a ProjectRelativePath { /// /// ``` - /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use std::convert::From; + /// + /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// /// let f = ForwardRelativePath::new("foo")?; /// - /// assert_eq!(<&ProjectRelativePath>::from(f), ProjectRelativePath::new("foo")?); + /// assert_eq!( + /// <&ProjectRelativePath>::from(f), + /// ProjectRelativePath::new("foo")? + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -504,6 +527,7 @@ impl<'a> TryFrom<&'a str> for &'a ProjectRelativePath { /// /// ``` /// use std::convert::TryFrom; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// @@ -525,6 +549,7 @@ impl<'a> TryFrom<&'a RelativePath> for &'a ProjectRelativePath { /// /// ``` /// use std::convert::TryFrom; + /// /// use buck2_core::fs::paths::RelativePath; /// use buck2_core::fs::project_rel_path::ProjectRelativePath; /// @@ -546,9 +571,10 @@ impl TryFrom for ProjectRelativePathBuf { /// no allocation conversion /// /// ``` - /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// use std::convert::TryFrom; /// + /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; + /// /// assert!(ProjectRelativePathBuf::try_from("foo/bar".to_owned()).is_ok()); /// assert!(ProjectRelativePathBuf::try_from("".to_owned()).is_ok()); /// assert!(ProjectRelativePathBuf::try_from("/abs/bar".to_owned()).is_err()); @@ -569,8 +595,9 @@ impl TryFrom for ProjectRelativePathBuf { /// conversion) /// /// ``` - /// use buck2_core::fs::paths::RelativePathBuf; /// use std::convert::TryFrom; + /// + /// use buck2_core::fs::paths::RelativePathBuf; /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// /// assert!(ProjectRelativePathBuf::try_from(RelativePathBuf::from("foo/bar")).is_ok()); @@ -591,9 +618,9 @@ impl TryFrom for ProjectRelativePathBuf { /// no allocation conversion /// /// ``` - /// /// use std::convert::TryFrom; /// use std::path::PathBuf; + /// /// use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; /// /// assert!(ProjectRelativePathBuf::try_from(PathBuf::from("foo/bar")).is_ok()); diff --git a/app/buck2_core/src/lib.rs b/app/buck2_core/src/lib.rs index 83b1b3fcc2603..75e0fb3efbae1 100644 --- a/app/buck2_core/src/lib.rs +++ b/app/buck2_core/src/lib.rs @@ -55,10 +55,19 @@ pub mod unsafe_send_future; /// en-mass at some point in the future. pub fn facebook_only() {} +/// Emit one expression or another depending on whether this is an open source or internal build. +#[macro_export] +macro_rules! if_else_opensource { + ($opensource:expr, $internal:expr $(,)? + ) => { + // @oss-disable: $internal + $opensource // @oss-enable + }; +} + #[inline] pub fn is_open_source() -> bool { - // @oss-disable: false - true // @oss-enable + if_else_opensource!(true, false) } /// Internal build with `buck2`. diff --git a/app/buck2_core/src/package/package_relative_path.rs b/app/buck2_core/src/package/package_relative_path.rs index d38a2186ee7e0..14a139391ce61 100644 --- a/app/buck2_core/src/package/package_relative_path.rs +++ b/app/buck2_core/src/package/package_relative_path.rs @@ -140,9 +140,10 @@ impl PackageRelativePath { /// normalized relative path, otherwise error. /// /// ``` - /// use buck2_core::package::package_relative_path::PackageRelativePath; /// use std::path::Path; /// + /// use buck2_core::package::package_relative_path::PackageRelativePath; + /// /// assert!(PackageRelativePath::new("foo/bar").is_ok()); /// assert!(PackageRelativePath::new("").is_ok()); /// assert!(PackageRelativePath::new("/abs/bar").is_err()); @@ -181,12 +182,17 @@ impl PackageRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use buck2_core::package::package_relative_path::{PackageRelativePath, PackageRelativePathBuf}; + /// use buck2_core::package::package_relative_path::PackageRelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// /// let path = PackageRelativePath::new("foo/bar")?; /// let other = ForwardRelativePath::new("baz")?; - /// assert_eq!(PackageRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), path.join(other)); + /// assert_eq!( + /// PackageRelativePathBuf::unchecked_new("foo/bar/baz".to_owned()), + /// path.join(other) + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -219,10 +225,13 @@ impl PackageRelativePath { /// a directory, this is the directory name. /// /// ``` - /// use buck2_core::package::package_relative_path::PackageRelativePath; /// use buck2_core::fs::paths::file_name::FileName; + /// use buck2_core::package::package_relative_path::PackageRelativePath; /// - /// assert_eq!(Some(FileName::unchecked_new("bin")), PackageRelativePath::new("usr/bin")?.file_name()); + /// assert_eq!( + /// Some(FileName::unchecked_new("bin")), + /// PackageRelativePath::new("usr/bin")?.file_name() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -245,7 +254,11 @@ impl PackageRelativePath { /// path.strip_prefix(PackageRelativePath::new("test")?)?, /// ForwardRelativePath::new("haha/foo.txt")? /// ); - /// assert_eq!(path.strip_prefix(PackageRelativePath::new("asdf")?).is_err(), true); + /// assert_eq!( + /// path.strip_prefix(PackageRelativePath::new("asdf")?) + /// .is_err(), + /// true + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -281,6 +294,7 @@ impl PackageRelativePath { /// /// ``` /// use std::path::Path; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; /// use buck2_core::package::package_relative_path::PackageRelativePath; /// @@ -322,10 +336,12 @@ impl PackageRelativePath { /// Extracts the extension of [`self.file_name`], if possible. /// /// ``` - /// /// use buck2_core::package::package_relative_path::PackageRelativePath; /// - /// assert_eq!(Some("rs"), PackageRelativePath::new("hi/foo.rs")?.extension()); + /// assert_eq!( + /// Some("rs"), + /// PackageRelativePath::new("hi/foo.rs")?.extension() + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -343,22 +359,10 @@ impl PackageRelativePath { /// let p = PackageRelativePath::new("foo/bar/baz")?; /// let mut it = p.iter(); /// - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("foo")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("bar")) - /// ); - /// assert_eq!( - /// it.next(), - /// Some(FileName::unchecked_new("baz")) - /// ); - /// assert_eq!( - /// it.next(), - /// None - /// ); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("foo"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("bar"))); + /// assert_eq!(it.next(), Some(FileName::unchecked_new("baz"))); + /// assert_eq!(it.next(), None); /// /// # anyhow::Ok(()) /// ``` @@ -386,14 +390,17 @@ impl PackageRelativePath { impl<'a> From<&'a ForwardRelativePath> for &'a PackageRelativePath { /// /// ``` + /// use std::convert::From; /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; - /// use std::convert::From; /// use buck2_core::package::package_relative_path::PackageRelativePath; /// /// let f = ForwardRelativePath::new("foo")?; /// - /// assert_eq!(<&PackageRelativePath>::from(f), PackageRelativePath::new("foo")?); + /// assert_eq!( + /// <&PackageRelativePath>::from(f), + /// PackageRelativePath::new("foo")? + /// ); /// /// # anyhow::Ok(()) /// ``` @@ -492,10 +499,10 @@ impl<'a> TryFrom<&'a str> for &'a PackageRelativePath { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::package::package_relative_path::PackageRelativePath; /// use std::convert::TryFrom; + /// /// use buck2_core::fs::paths::forward_rel_path::ForwardRelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePath; /// /// assert!(<&PackageRelativePath>::try_from("foo/bar").is_ok()); /// assert!(<&PackageRelativePath>::try_from("").is_ok()); @@ -515,10 +522,10 @@ impl<'a> TryFrom<&'a RelativePath> for &'a PackageRelativePath { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::package::package_relative_path::PackageRelativePath; /// use std::convert::TryFrom; + /// /// use buck2_core::fs::paths::RelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePath; /// /// assert!(<&PackageRelativePath>::try_from(RelativePath::new("foo/bar")).is_ok()); /// assert!(<&PackageRelativePath>::try_from(RelativePath::new("")).is_ok()); @@ -539,10 +546,10 @@ impl TryFrom for PackageRelativePathBuf { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// use std::convert::TryFrom; + /// /// use buck2_core::package::package_relative_path::PackageRelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// /// assert!(PackageRelativePathBuf::try_from("foo/bar".to_owned()).is_ok()); /// assert!(PackageRelativePathBuf::try_from("".to_owned()).is_ok()); @@ -565,10 +572,11 @@ impl TryFrom for PackageRelativePathBuf { /// conversion) /// /// ``` - /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; - /// use buck2_core::fs::paths::RelativePathBuf; /// use std::convert::TryFrom; + /// + /// use buck2_core::fs::paths::RelativePathBuf; /// use buck2_core::package::package_relative_path::PackageRelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// /// assert!(PackageRelativePathBuf::try_from(RelativePathBuf::from("foo/bar")).is_ok()); /// assert!(PackageRelativePathBuf::try_from(RelativePathBuf::from("")).is_ok()); @@ -589,11 +597,11 @@ impl TryFrom for PackageRelativePathBuf { /// no allocation conversion /// /// ``` - /// - /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// use std::convert::TryFrom; /// use std::path::PathBuf; + /// /// use buck2_core::package::package_relative_path::PackageRelativePath; + /// use buck2_core::package::package_relative_path::PackageRelativePathBuf; /// /// assert!(PackageRelativePathBuf::try_from(PathBuf::from("foo/bar")).is_ok()); /// assert!(PackageRelativePathBuf::try_from(PathBuf::from("")).is_ok()); diff --git a/app/buck2_core/src/pattern/mod.rs b/app/buck2_core/src/pattern/mod.rs index b4ee32f7536e6..6b3c0cf2dd181 100644 --- a/app/buck2_core/src/pattern/mod.rs +++ b/app/buck2_core/src/pattern/mod.rs @@ -8,7 +8,6 @@ */ //! Implements target pattern resolution. -//! #![doc = include_str!("target_pattern.md")] mod ascii_pattern; diff --git a/app/buck2_core/src/pattern/target_pattern.md b/app/buck2_core/src/pattern/target_pattern.md index 2d2d407db50bb..7c9d8c9669500 100644 --- a/app/buck2_core/src/pattern/target_pattern.md +++ b/app/buck2_core/src/pattern/target_pattern.md @@ -1,34 +1,60 @@ -A __target pattern__ is a string that describes a set of one or more targets. You can use target patterns as arguments to commands, such as buck build and buck query. You can also use target patterns in the Visibility argument of your build rules. - -The simplest target pattern `//apps/myapp:app` matches exactly the target of the same name `//apps/myapp:app`. - - -A target pattern that ends with a colon matches all targets in the build file at the preceding directory path. For example, suppose that the build file `apps/myapp/BUCK` defines the rules: app_debug and app_release, then the target pattern `//apps/myapp:` matches `//apps/myapp:app_debug` and `//apps/myapp:app_release`. - - -A target pattern that ends with an ellipsis "/..." matches all targets in the build file in the directory that precedes the ellipsis and also all targets in build files in subdirectories (within the same cell). For example, suppose that you have the following build files: `apps/BUCK`, `apps/myapp/BUCK`. Then the target pattern `//apps/...` would match (for example) `//apps:common` and `//apps/myapp:app`. The pattern `//...` would match the same (even though there's no build file in the root directory). +A **target pattern** is a string that describes a set of one or more targets. +You can use target patterns as arguments to commands, such as buck build and +buck query. You can also use target patterns in the Visibility argument of your +build rules. + +The simplest target pattern `//apps/myapp:app` matches exactly the target of the +same name `//apps/myapp:app`. + +A target pattern that ends with a colon matches all targets in the build file at +the preceding directory path. For example, suppose that the build file +`apps/myapp/BUCK` defines the rules: app_debug and app_release, then the target +pattern `//apps/myapp:` matches `//apps/myapp:app_debug` and +`//apps/myapp:app_release`. + +A target pattern that ends with an ellipsis "/..." matches all targets in the +build file in the directory that precedes the ellipsis and also all targets in +build files in subdirectories (within the same cell). For example, suppose that +you have the following build files: `apps/BUCK`, `apps/myapp/BUCK`. Then the +target pattern `//apps/...` would match (for example) `//apps:common` and +`//apps/myapp:app`. The pattern `//...` would match the same (even though +there's no build file in the root directory). ## Cell resolution -Cells will be resolved in the context where the target pattern appears. When used as arguments to the command line, they will be resolved based on the cell of the directory in which the command is invoked. +Cells will be resolved in the context where the target pattern appears. When +used as arguments to the command line, they will be resolved based on the cell +of the directory in which the command is invoked. -If `~/project` and `~/project/cell` are both cells with names `project` and `cell` respectively, then `//some:target` would resolve to `project//some:target` if it appears in `~/project/BUCK` and `cell//some:target` if it appears in `~/project/cell/BUCK`. +If `~/project` and `~/project/cell` are both cells with names `project` and +`cell` respectively, then `//some:target` would resolve to +`project//some:target` if it appears in `~/project/BUCK` and `cell//some:target` +if it appears in `~/project/cell/BUCK`. ## Relative patterns -Target patterns can be absolute (`//my/app:target`, `cell//other:target`) or relative `app:target`. A relative pattern will be resolved relative to the working directory of the command. +Target patterns can be absolute (`//my/app:target`, `cell//other:target`) or +relative `app:target`. A relative pattern will be resolved relative to the +working directory of the command. ## Restrictions -A target pattern should not include any `..` segments. This applies to both absolute and relative patterns. +A target pattern should not include any `..` segments. This applies to both +absolute and relative patterns. ## Inner Providers -A target pattern used where providers labels are expected can refer to rule's inner providers via `//my/app:target[]` syntax. -The inner providers label will refer to a specific set of providers exposed by a rule, such as a particular set of outputs from the rule. -The providers label can be used for commands: buck builds, provider queries, and action queries. -Any rule's dependencies also refers to a providers label. -However, configuration rules (i.e config_settings) should be referred to without providers. -(TODO: experiment and see if we should just use provider labels everywhere). +A target pattern used where providers labels are expected can refer to rule's +inner providers via `//my/app:target[]` syntax. The inner +providers label will refer to a specific set of providers exposed by a rule, +such as a particular set of outputs from the rule. + +The providers label can be used for commands: buck builds, provider queries, and +action queries. Any rule's dependencies also refers to a providers label. +However, configuration rules (i.e config_settings) should be referred to without +providers. (TODO: experiment and see if we should just use provider labels +everywhere). -The providers label syntax can only be used when the pattern is of a specific rule. Package and recursive patterns (e.g. `//some/pkg:` and `//some/...`) cannot have providers labels. +The providers label syntax can only be used when the pattern is of a specific +rule. Package and recursive patterns (e.g. `//some/pkg:` and `//some/...`) +cannot have providers labels. diff --git a/app/buck2_core/src/target/configured_or_unconfigured.rs b/app/buck2_core/src/target/configured_or_unconfigured.rs new file mode 100644 index 0000000000000..1fddf2ca3ce98 --- /dev/null +++ b/app/buck2_core/src/target/configured_or_unconfigured.rs @@ -0,0 +1,27 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use crate::target::configured_target_label::ConfiguredTargetLabel; +use crate::target::label::TargetLabel; + +pub trait ConfiguredOrUnconfiguredTargetLabel { + fn unconfigured_label(&self) -> &TargetLabel; +} + +impl ConfiguredOrUnconfiguredTargetLabel for TargetLabel { + fn unconfigured_label(&self) -> &TargetLabel { + self + } +} + +impl ConfiguredOrUnconfiguredTargetLabel for ConfiguredTargetLabel { + fn unconfigured_label(&self) -> &TargetLabel { + self.unconfigured() + } +} diff --git a/app/buck2_core/src/target/configured_target_label.rs b/app/buck2_core/src/target/configured_target_label.rs index d3a27a947f5f8..b4a9b7a1cb17f 100644 --- a/app/buck2_core/src/target/configured_target_label.rs +++ b/app/buck2_core/src/target/configured_target_label.rs @@ -9,6 +9,7 @@ use std::fmt; use std::fmt::Display; +use std::hash::Hash; use std::str; use allocative::Allocative; diff --git a/app/buck2_core/src/target/mod.rs b/app/buck2_core/src/target/mod.rs index 435b98b99272f..1fd3e13dd350c 100644 --- a/app/buck2_core/src/target/mod.rs +++ b/app/buck2_core/src/target/mod.rs @@ -34,6 +34,7 @@ //! `my/package/path` is the package, and `my_target` is the target name //! belonging to the package. +pub mod configured_or_unconfigured; pub mod configured_target_label; pub mod label; pub mod name; diff --git a/app/buck2_core/tests/soft_error.rs b/app/buck2_core/tests/soft_error.rs index e53731680ccc2..9b4220e2b35a2 100644 --- a/app/buck2_core/tests/soft_error.rs +++ b/app/buck2_core/tests/soft_error.rs @@ -65,7 +65,7 @@ fn test_soft_error() { file!(), before_error_line + 1, )), - RESULT.lock().unwrap().get(0) + RESULT.lock().unwrap().first() ); } diff --git a/app/buck2_critical_path/src/builder.rs b/app/buck2_critical_path/src/builder.rs index 9757082ca4bb5..7bbb52a5bb8fa 100644 --- a/app/buck2_critical_path/src/builder.rs +++ b/app/buck2_critical_path/src/builder.rs @@ -115,7 +115,7 @@ where } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_critical_path/src/graph.rs b/app/buck2_critical_path/src/graph.rs index d2f01251b8cd2..ca1c5439a1c6f 100644 --- a/app/buck2_critical_path/src/graph.rs +++ b/app/buck2_critical_path/src/graph.rs @@ -279,7 +279,7 @@ pub struct PathCost { } #[cfg(test)] -mod test { +mod tests { use super::*; use crate::builder::GraphBuilder; use crate::test_utils::make_dag; diff --git a/app/buck2_critical_path/src/potential.rs b/app/buck2_critical_path/src/potential.rs index 54aa051954e8b..fe22885959119 100644 --- a/app/buck2_critical_path/src/potential.rs +++ b/app/buck2_critical_path/src/potential.rs @@ -281,7 +281,7 @@ pub fn compute_critical_path_potentials( } #[cfg(test)] -mod test { +mod tests { use std::time::Instant; use rand::SeedableRng; diff --git a/app/buck2_data/build.rs b/app/buck2_data/build.rs index c839fb8c568f4..a4c157a1021d9 100644 --- a/app/buck2_data/build.rs +++ b/app/buck2_data/build.rs @@ -186,6 +186,10 @@ fn main() -> io::Result<()> { "CriticalPathEntry2.potential_improvement_duration", "#[serde(rename = \"potential_improvement_duration_us\", with = \"crate::serialize_duration_as_micros\")]", ) + .field_attribute( + "CriticalPathEntry2.queue_duration", + "#[serde(rename = \"queue_duration_us\", with = \"crate::serialize_duration_as_micros\")]", + ) .type_attribute( "buck.data.CriticalPathEntry2.entry", "#[derive(::derive_more::From, ::gazebo::variants::VariantName)]", @@ -242,6 +246,10 @@ fn main() -> io::Result<()> { "buck.data.CommandExecutionMetadata.hashing_duration", "#[serde(rename = \"hashing_duration_us\", with = \"crate::serialize_duration_as_micros\")]", ) + .field_attribute( + "buck.data.CommandExecutionMetadata.queue_duration", + "#[serde(rename = \"queue_duration_us\", with = \"crate::serialize_duration_as_micros\")]", + ) .boxed("RecordEvent.data.invocation_record") .boxed("SpanEndEvent.data.action_execution") .boxed("SpanEndEvent.data.cache_upload") diff --git a/app/buck2_data/data.proto b/app/buck2_data/data.proto index fb2dd9ff92fd9..63bc144fadfea 100644 --- a/app/buck2_data/data.proto +++ b/app/buck2_data/data.proto @@ -85,6 +85,7 @@ message SpanStartEvent { BxlEnsureArtifactsStart bxl_ensure_artifacts = 82; CreateOutputHashesFileStart create_output_hashes_file = 84; ActionErrorHandlerExecutionStart action_error_handler_execution = 85; + CqueryUniverseBuildStart cquery_universe_build = 87; // Used in Buck unit tests. FakeStart fake = 999; } @@ -134,6 +135,7 @@ message SpanEndEvent { BxlEnsureArtifactsEnd bxl_ensure_artifacts = 83; CreateOutputHashesFileEnd create_output_hashes_file = 85; ActionErrorHandlerExecutionEnd action_error_handler_execution = 86; + CqueryUniverseBuildEnd cquery_universe_build = 87; // Used in Buck unit tests. FakeEnd fake = 999; } @@ -248,7 +250,7 @@ message InstantEvent { ConcurrentCommands concurrent_commands = 32; // Info coming from the `buck2 debug persist-event-log` subprocess - PersistSubprocess persist_subprocess = 33; + PersistEventLogSubprocess persist_event_log_subprocess = 33; // An action error encountered during the build ActionError action_error = 34; @@ -423,6 +425,10 @@ message CriticalPathEntry2 { // `duration` (since it can't exceed it). optional google.protobuf.Duration potential_improvement_duration = 5; + // The subset of the duration that can be attributed to waiting + // for actions to run + optional google.protobuf.Duration queue_duration = 6; + oneof entry { Analysis analysis = 100; ActionExecution action_execution = 101; @@ -995,10 +1001,15 @@ message CommandExecutionMetadata { CommandExecutionStats execution_stats = 5; - /// How long it took to hash the action's artifacts + /// Sum of all hashing times of individual files. Can be larger than user time + /// because hashing is done in parallel. + google.protobuf.Duration hashing_duration = 6; uint64 hashed_artifacts_count = 7; + + /// How long this command spent waiting to run + optional google.protobuf.Duration queue_duration = 8; } message CommandOutputsMissing { @@ -1214,6 +1225,10 @@ message ActionErrorHandlerExecutionStart {} message ActionErrorHandlerExecutionEnd {} +message CqueryUniverseBuildStart {} + +message CqueryUniverseBuildEnd {} + // The beginning of materialization for the output of a target requested, // inclusive of all dependent artifacts it might recursively request to // materialize. @@ -1633,6 +1648,18 @@ message TypedMetadata { map strings = 2; } +// Why current command started buckd. +enum DaemonWasStartedReason { + UNKNOWN_REASON = 0; + CONSTRAINT_MISMATCH_VERSION = 1; + CONSTRAINT_MISMATCH_USER_VERSION = 2; + CONSTRAINT_MISMATCH_STARTUP_CONFIG = 3; + CONSTRAINT_REJECT_DAEMON_ID = 4; + CONSTRAINT_MISMATCH_TRACE_IO = 5; + CONSTRAINT_MISMATCH_MATERIALIZER_STATE_IDENTITY = 6; + COULD_NOT_CONNECT_TO_DAEMON = 11; +} + // This is the origin for every sample in buck2_builds scuba table // It's sent from the client to Scribe at the end of each invocation message InvocationRecord { @@ -1754,11 +1781,17 @@ message InvocationRecord { repeated string concurrent_command_ids = 74; // The client has failed to connect to the daemon. optional bool daemon_connection_failure = 75; + // Daemon was started by this command. + // Unset if the the command connected to existing daemon or did not start one. + optional DaemonWasStartedReason daemon_was_started = 751; // Metadata provided by the client. Unlike TypedMetadata, this won't become // its own column in Scuba, all those entries will land in a NormVector. repeated ClientMetadata client_metadata = 76; // The errors that occured during the command. repeated ProcessedErrorReport errors = 79; + // The most interesting error tag that occurred during the command. + // Uppercase of `ErrorTag` enum variant, e.g. `STARLARK_FAIL`. + optional string best_error_tag = 82; // Cache hit rate as it appears in the console float cache_hit_rate = 78; repeated string target_rule_type_names = 80; @@ -1953,6 +1986,9 @@ message ProcessedErrorReport { optional string telemetry_message = 4; optional string source_location = 5; repeated string tags = 6; + // `buck2_error` crate has logic of selecting the most interesting error tag + // among all error tags. This is such tag. + optional string best_tag = 7; } message MaterializerStateInfo { @@ -1968,8 +2004,15 @@ message ConcurrentCommands { repeated string trace_ids = 1; } -message PersistSubprocess { - repeated string errors = 1; +message PersistEventLogSubprocess { + repeated string local_error_messages = 1; + optional string local_error_category = 2; + bool local_success = 3; + repeated string remote_error_messages = 4; + optional string remote_error_category = 5; + bool remote_success = 6; + bool allow_vpnless = 7; + map metadata = 8; } message StarlarkFailNoStacktrace { diff --git a/app/buck2_data/error.proto b/app/buck2_data/error.proto index 991df1425fba3..529bab9f40809 100644 --- a/app/buck2_data/error.proto +++ b/app/buck2_data/error.proto @@ -16,6 +16,8 @@ syntax = "proto3"; // elsewhere. package buck.data.error; +// TODO(jakobdegen): this enum and `error_subcategory` scuba column +// are deprecated. Use category+tag. enum ErrorType { // Protobuf requires us to supply a default value; however, this type is // always used in an `optional` way and so no default value should ever @@ -24,6 +26,7 @@ enum ErrorType { DAEMON_IS_BUSY = 1; ACTION_COMMAND_FAILURE = 2; WATCHMAN = 3; + USER_DEADLINE_EXPIRED = 4; // Add causes here as needed } @@ -46,4 +49,30 @@ enum ErrorTag { UNUSED_DEFAULT_TAG = 0; STARLARK_FAIL = 1; WATCHMAN_TIMEOUT = 2; + HTTP = 3; + // gRPC protocol error between client and server from the client side. + // - Protocol error (e.g. malformed frame, or too large frame) + // - Transport error (e.g. connection closed) + // - Not application error (e.g. bzl file not found) + CLIENT_GRPC = 4; + // Connect to buckd failed. + DAEMON_CONNECT = 5; + // Too large gRPC message. + GRPC_RESPONSE_MESSAGE_TOO_LARGE = 6; + // Error during analysis. + ANALYSIS = 7; + // `visibility`, `within_view`. + VISIBILITY = 8; + // Server stderr is empty. + SERVER_STDERR_EMPTY = 11; + // Server stderr indicates that the server panicked. + SERVER_PANICKED = 12; + // Server stack overflow. + SERVER_STACK_OVERFLOW = 13; + // SEGV. + SERVER_SEGV = 14; + // Jemalloc assertion failure. + SERVER_JEMALLOC_ASSERT = 15; + // The reason for server failure is unknown. + SERVER_STDERR_UNKNOWN = 19; } diff --git a/app/buck2_data/src/lib.rs b/app/buck2_data/src/lib.rs index 2ac537981ab99..0cf1a51f49ba1 100644 --- a/app/buck2_data/src/lib.rs +++ b/app/buck2_data/src/lib.rs @@ -180,7 +180,7 @@ pub mod serialize_duration_as_micros { } #[cfg(test)] - mod test { + mod tests { use super::*; #[test] diff --git a/app/buck2_error/src/any.rs b/app/buck2_error/src/any.rs index edefbbfa76949..82cfbc1ed30ce 100644 --- a/app/buck2_error/src/any.rs +++ b/app/buck2_error/src/any.rs @@ -249,7 +249,7 @@ mod tests { Some("buck2_error/src/any.rs::FullMetadataError") ); assert_eq!( - &e.get_tags(), + &e.tags(), &[ crate::ErrorTag::StarlarkFail, crate::ErrorTag::WatchmanTimeout diff --git a/app/buck2_error/src/classify.rs b/app/buck2_error/src/classify.rs new file mode 100644 index 0000000000000..22d714d670d76 --- /dev/null +++ b/app/buck2_error/src/classify.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use buck2_data::error::ErrorTag; + +/// When there's no tag, but we want to put something in Scuba, we use this. +pub const ERROR_TAG_UNCLASSIFIED: &str = "UNCLASSIFIED"; + +/// Pick the most interesting tag from a list of tags. +pub fn best_tag(tags: impl IntoIterator) -> Option { + tags.into_iter().min_by_key(|t| tag_rank(*t)) +} + +/// Tag rank: smaller is more interesting. +fn tag_rank(tag: ErrorTag) -> u32 { + match tag { + ErrorTag::ServerJemallocAssert => line!(), + ErrorTag::ServerStackOverflow => line!(), + ErrorTag::ServerPanicked => line!(), + ErrorTag::ServerSegv => line!(), + ErrorTag::DaemonConnect => line!(), + ErrorTag::GrpcResponseMessageTooLarge => line!(), + ErrorTag::ClientGrpc => line!(), + ErrorTag::StarlarkFail => line!(), + ErrorTag::Visibility => line!(), + ErrorTag::Analysis => line!(), + ErrorTag::WatchmanTimeout => line!(), + ErrorTag::Http => line!(), + ErrorTag::ServerStderrUnknown => line!(), + ErrorTag::ServerStderrEmpty => line!(), + ErrorTag::UnusedDefaultTag => line!(), + } +} + +#[cfg(test)] +mod tests { + use buck2_data::error::ErrorTag; + + use crate::classify::best_tag; + + #[test] + fn test_best_tag() { + assert_eq!( + Some(ErrorTag::ServerPanicked), + best_tag([ErrorTag::ServerPanicked, ErrorTag::WatchmanTimeout]) + ) + } +} diff --git a/app/buck2_error/src/context.rs b/app/buck2_error/src/context.rs index f4d72857564ed..9566ad521e99f 100644 --- a/app/buck2_error/src/context.rs +++ b/app/buck2_error/src/context.rs @@ -7,32 +7,10 @@ * of this source tree. */ -use std::sync::Arc; - use smallvec::smallvec; use crate as buck2_error; use crate::context_value::ContextValue; -use crate::error::ErrorKind; - -impl crate::Error { - pub fn context>(self, context: C) -> Self { - Self(Arc::new(ErrorKind::WithContext(context.into(), self))) - } - - #[cold] - #[track_caller] - fn new_anyhow_with_context>(e: E, c: C) -> anyhow::Error - where - crate::Error: From, - { - crate::Error::from(e).context(c).into() - } - - pub fn tag(self, tags: impl IntoIterator) -> Self { - self.context(ContextValue::Tags(tags.into_iter().collect())) - } -} /// Provides the `context` method for `Result`. /// diff --git a/app/buck2_error/src/context_value.rs b/app/buck2_error/src/context_value.rs index 289b11e676d91..a4c79f54e49c1 100644 --- a/app/buck2_error/src/context_value.rs +++ b/app/buck2_error/src/context_value.rs @@ -62,41 +62,25 @@ impl From for ContextValue { } } -#[derive(allocative::Allocative, PartialEq, Eq, Copy, Clone, Debug)] +#[derive( + allocative::Allocative, + PartialEq, + Eq, + Copy, + Clone, + Debug, + PartialOrd, + Ord +)] pub enum Category { User, Infra, } -impl crate::Error { - pub fn get_category(&self) -> Option { - let mut out = None; - for cat in self.iter_context().filter_map(|kind| match kind { - ContextValue::Category(cat) => Some(*cat), - _ => None, - }) { - // It's an infra error if it was ever marked as an infra error - match cat { - Category::Infra => return Some(cat), - Category::User => out = Some(cat), - } - } - out - } - - /// Get all the tags that have been added to this error - pub fn get_tags(&self) -> Vec { - let mut tags: Vec<_> = self - .iter_context() - .filter_map(|kind| match kind { - ContextValue::Tags(tags) => Some(tags.iter().copied()), - _ => None, - }) - .flatten() - .collect(); - tags.sort_unstable_by_key(|tag| tag.as_str_name()); - tags.dedup(); - tags +impl Category { + pub fn combine(self, other: Option) -> Category { + let Some(other) = other else { return self }; + std::cmp::max(self, other) } } @@ -108,7 +92,8 @@ impl From for ContextValue { #[cfg(test)] mod tests { - use crate as buck2_error; + use crate::Category; + use crate::{self as buck2_error}; #[derive(buck2_error_derive::Error, Debug)] #[error("test error")] @@ -131,4 +116,18 @@ mod tests { .context(crate::Category::User); assert_eq!(e.get_category(), Some(crate::Category::Infra)); } + + #[test] + fn test_combine() { + assert_eq!(Category::User.combine(None), Category::User); + assert_eq!(Category::User.combine(Some(Category::User)), Category::User); + assert_eq!( + Category::User.combine(Some(Category::Infra)), + Category::Infra + ); + assert_eq!( + Category::Infra.combine(Some(Category::User)), + Category::Infra + ); + } } diff --git a/app/buck2_error/src/derive_tests.rs b/app/buck2_error/src/derive_tests.rs index 8456760decaa8..9a184e2697dae 100644 --- a/app/buck2_error/src/derive_tests.rs +++ b/app/buck2_error/src/derive_tests.rs @@ -236,14 +236,14 @@ fn test_error_tags() { let a: crate::Error = TaggedError::A.into(); assert_eq!( - &a.get_tags(), + &a.tags(), &[ crate::ErrorTag::StarlarkFail, crate::ErrorTag::WatchmanTimeout ] ); let b: crate::Error = TaggedError::B.into(); - assert_eq!(&b.get_tags(), &[crate::ErrorTag::WatchmanTimeout]); + assert_eq!(&b.tags(), &[crate::ErrorTag::WatchmanTimeout]); } #[test] diff --git a/app/buck2_error/src/error.rs b/app/buck2_error/src/error.rs index b904c4775c6a8..2f8d56075ab9b 100644 --- a/app/buck2_error/src/error.rs +++ b/app/buck2_error/src/error.rs @@ -11,9 +11,11 @@ use std::error::Error as StdError; use std::fmt; use std::sync::Arc; +use crate::classify::best_tag; use crate::context_value::ContextValue; use crate::format::into_anyhow_for_format; use crate::root::ErrorRoot; +use crate::Category; use crate::ErrorType; use crate::UniqueRootId; @@ -136,6 +138,61 @@ impl Error { pub fn source_location(&self) -> Option<&str> { self.root().source_location() } + + pub fn context>(self, context: C) -> Self { + Self(Arc::new(ErrorKind::WithContext(context.into(), self))) + } + + #[cold] + #[track_caller] + pub(crate) fn new_anyhow_with_context>(e: E, c: C) -> anyhow::Error + where + Error: From, + { + crate::Error::from(e).context(c).into() + } + + pub fn tag(self, tags: impl IntoIterator) -> Self { + self.context(ContextValue::Tags(tags.into_iter().collect())) + } + + pub fn get_category(&self) -> Option { + let mut out = None; + for cat in self.iter_context().filter_map(|kind| match kind { + ContextValue::Category(cat) => Some(*cat), + _ => None, + }) { + // It's an infra error if it was ever marked as an infra error + match cat { + Category::Infra => return Some(cat), + Category::User => out = Some(cat), + } + } + out + } + + /// All tags unsorted and with duplicates. + fn tags_unsorted(&self) -> impl Iterator + '_ { + self.iter_context() + .filter_map(|kind| match kind { + ContextValue::Tags(tags) => Some(tags.iter().copied()), + _ => None, + }) + .flatten() + } + + /// Get all the tags that have been added to this error + pub fn tags(&self) -> Vec { + let mut tags: Vec<_> = self.tags_unsorted().collect(); + tags.sort_unstable_by_key(|tag| tag.as_str_name()); + tags.dedup(); + tags + } + + /// The most interesting tag among this error tags. + pub fn best_tag(&self) -> Option { + best_tag(self.tags_unsorted()) + } } #[cfg(test)] diff --git a/app/buck2_error/src/format.rs b/app/buck2_error/src/format.rs index c57f97c080b34..5f532e607c1d6 100644 --- a/app/buck2_error/src/format.rs +++ b/app/buck2_error/src/format.rs @@ -135,6 +135,7 @@ Caused by: 0: context 1 1: test error"#, ); + assert_eq_no_backtrace(format!("{:#}", e), r#"context 2: context 1: test error"#); } #[test] @@ -158,9 +159,31 @@ Caused by: let e2 = anyhow::Error::from(e.clone()); assert_eq_no_backtrace(format!("{}", e), format!("{}", e2)); assert_eq_no_backtrace(format!("{:?}", e), format!("{:?}", e2)); + assert_eq_no_backtrace(format!("{:#}", e), format!("{:#}", e2)); let e3 = buck2_error::Error::from(e2); assert_eq_no_backtrace(format!("{}", e), format!("{}", e3)); assert_eq_no_backtrace(format!("{:?}", e), format!("{:?}", e3)); + assert_eq_no_backtrace(format!("{:#}", e), format!("{:#}", e3)); + } + + #[test] + fn test_with_context_from_source() { + #[derive(buck2_error::Error, Debug)] + #[error("with source")] + struct E(#[source] TestError); + + let e = buck2_error::Error::from(E(TestError)).context("context"); + + assert_eq_no_backtrace( + format!("{:?}", e), + r#"context + +Caused by: + 0: with source + 1: test error"#, + ); + assert_eq_no_backtrace(format!("{:#}", e), r#"context: with source: test error"#); + assert_eq_no_backtrace(format!("{}", e), r#"context"#); } } diff --git a/app/buck2_error/src/lib.rs b/app/buck2_error/src/lib.rs index 799a65ae96f94..397901e23098f 100644 --- a/app/buck2_error/src/lib.rs +++ b/app/buck2_error/src/lib.rs @@ -13,6 +13,7 @@ #![feature(trait_upcasting)] mod any; +pub mod classify; mod context; mod context_value; mod derive_tests; @@ -39,6 +40,12 @@ pub use root::UniqueRootId; pub type Result = std::result::Result; +/// Allows simpler construction of the Ok case when the result type can't be inferred. +#[allow(non_snake_case)] +pub fn Ok(t: T) -> Result { + Result::Ok(t) +} + /// See the documentation in the `error.proto` file for details. pub use buck2_data::error::ErrorTag; /// The type of the error that is being produced. diff --git a/app/buck2_error_derive/src/expand.rs b/app/buck2_error_derive/src/expand.rs index 806a4deaed0dc..07459e3dfa848 100644 --- a/app/buck2_error_derive/src/expand.rs +++ b/app/buck2_error_derive/src/expand.rs @@ -306,38 +306,42 @@ fn gen_provide_contents( fields: &[Field], type_name: &Ident, variant_name: Option<&Ident>, -) -> TokenStream { +) -> syn::Stmt { let type_and_variant = match variant_name { Some(variant_name) => format!("{}::{}", type_name, variant_name), None => type_name.to_string(), }; let source_location_extra = syn::LitStr::new(&type_and_variant, Span::call_site()); - let category = match &attrs.category { - Some(OptionStyle::Explicit(cat)) => quote::quote! { + let category: syn::Expr = match &attrs.category { + Some(OptionStyle::Explicit(cat)) => syn::parse_quote! { core::option::Option::Some(buck2_error::Category::#cat) }, - Some(OptionStyle::ByExpr(e)) => e.to_token_stream(), - None => quote::quote! { + Some(OptionStyle::ByExpr(e)) => e.clone(), + None => syn::parse_quote! { core::option::Option::None }, }; - let typ = match &attrs.typ { - Some(OptionStyle::Explicit(typ)) => quote::quote! { + let typ: syn::Expr = match &attrs.typ { + Some(OptionStyle::Explicit(typ)) => syn::parse_quote! { core::option::Option::Some(buck2_error::ErrorType::#typ) }, - Some(OptionStyle::ByExpr(e)) => e.to_token_stream(), - None => quote::quote! { + Some(OptionStyle::ByExpr(e)) => e.clone(), + None => syn::parse_quote! { core::option::Option::None }, }; - let tags = attrs.tags.iter().map(|tag| match tag { - OptionStyle::Explicit(tag) => quote::quote! { - core::option::Option::Some(buck2_error::ErrorTag::#tag) - }, - OptionStyle::ByExpr(e) => e.to_token_stream(), - }); + let tags: Vec = attrs + .tags + .iter() + .map(|tag| match tag { + OptionStyle::Explicit(tag) => syn::parse_quote! { + core::option::Option::Some(buck2_error::ErrorTag::#tag) + }, + OptionStyle::ByExpr(e) => e.clone(), + }) + .collect(); - let metadata = quote! { + let metadata: syn::Stmt = syn::parse_quote! { buck2_error::provide_metadata( __request, #category, @@ -364,9 +368,11 @@ fn gen_provide_contents( // When the same type is provided to the `request` more than once, the first value is used and // later values are ignored. As such, make sure we put the `forward_transparent` first, so that // if the underlying error has metadata, that's the one that gets used - quote! { - #forward_transparent - #metadata + syn::parse_quote! { + { + #forward_transparent + #metadata + } } } diff --git a/app/buck2_error_derive/src/lib.rs b/app/buck2_error_derive/src/lib.rs index f3df100ef875e..d0a118802fc39 100644 --- a/app/buck2_error_derive/src/lib.rs +++ b/app/buck2_error_derive/src/lib.rs @@ -10,7 +10,7 @@ // This code is adapted from https://github.com/dtolnay/thiserror licensed under Apache-2.0 or MIT. #![allow( - clippy::blocks_in_if_conditions, + clippy::blocks_in_conditions, clippy::cast_lossless, clippy::cast_possible_truncation, clippy::manual_find, diff --git a/app/buck2_event_log/src/utils.rs b/app/buck2_event_log/src/utils.rs index 173700a3aa945..9972a901a7aad 100644 --- a/app/buck2_event_log/src/utils.rs +++ b/app/buck2_event_log/src/utils.rs @@ -144,11 +144,13 @@ pub struct Invocation { impl Invocation { pub fn display_command_line(&self) -> String { - shlex::join(self.command_line_args.iter().map(|e| e.as_str())) + shlex::try_join(self.command_line_args.iter().map(|e| e.as_str())) + .expect("Null byte unexpected") } pub fn display_expanded_command_line(&self) -> String { - shlex::join(self.expanded_command_line_args.iter().map(|e| e.as_str())) + shlex::try_join(self.expanded_command_line_args.iter().map(|e| e.as_str())) + .expect("Null byte unexpected") } pub(crate) fn parse_json_line(json: &str) -> anyhow::Result { diff --git a/app/buck2_event_log/src/write.rs b/app/buck2_event_log/src/write.rs index aed74f9a0b09c..7012a34bfaf8b 100644 --- a/app/buck2_event_log/src/write.rs +++ b/app/buck2_event_log/src/write.rs @@ -272,7 +272,7 @@ impl<'a> WriteEventLog<'a> { path: logdir.as_abs_path().join(file_name), encoding, }; - let writer = start_persist_subprocess( + let writer = start_persist_event_log_subprocess( path, event.trace_id()?.clone(), self.log_size_counter_bytes.clone(), @@ -368,7 +368,7 @@ impl<'a> Drop for WriteEventLog<'a> { } } -async fn start_persist_subprocess( +async fn start_persist_event_log_subprocess( path: EventLogPathBuf, trace_id: TraceId, bytes_written: Option>, diff --git a/app/buck2_event_observer/src/display.rs b/app/buck2_event_observer/src/display.rs index 679bde22d74ef..a232df0a23ff5 100644 --- a/app/buck2_event_observer/src/display.rs +++ b/app/buck2_event_observer/src/display.rs @@ -343,6 +343,7 @@ pub fn display_event(event: &BuckEvent, opts: TargetDisplayOptions) -> anyhow::R Data::ActionErrorHandlerExecution(..) => { Ok("Running error handler on action failure".to_owned()) } + Data::CqueryUniverseBuild(..) => Ok("Building cquery universe".to_owned()), }; // This shouldn't really be necessary, but that's how try blocks work :( diff --git a/app/buck2_event_observer/src/humanized.rs b/app/buck2_event_observer/src/humanized.rs index eb820b0b2eb16..8823bde00f363 100644 --- a/app/buck2_event_observer/src/humanized.rs +++ b/app/buck2_event_observer/src/humanized.rs @@ -169,7 +169,7 @@ impl fmt::Display for HumanizedCount { } #[cfg(test)] -mod test { +mod tests { use super::HumanizedBytes; use super::HumanizedBytesPerSecond; use super::HumanizedCount; diff --git a/app/buck2_event_observer/src/span_tracker.rs b/app/buck2_event_observer/src/span_tracker.rs index c2a763f221e35..7391af1a5a09f 100644 --- a/app/buck2_event_observer/src/span_tracker.rs +++ b/app/buck2_event_observer/src/span_tracker.rs @@ -542,7 +542,8 @@ pub fn is_span_shown(event: &BuckEvent) -> bool { | Data::LocalResources(..) | Data::ReleaseLocalResources(..) | Data::CreateOutputHashesFile(..) - | Data::ActionErrorHandlerExecution(..), + | Data::ActionErrorHandlerExecution(..) + | Data::CqueryUniverseBuild(..), ) => true, None => false, } @@ -598,7 +599,7 @@ impl fmt::Display for OptionalSpanId { } #[cfg(test)] -mod test { +mod tests { use std::sync::atomic::AtomicI64; use std::sync::atomic::Ordering; diff --git a/app/buck2_event_observer/src/what_ran.rs b/app/buck2_event_observer/src/what_ran.rs index 1e74ab65b6c9d..b0744cca0c00f 100644 --- a/app/buck2_event_observer/src/what_ran.rs +++ b/app/buck2_event_observer/src/what_ran.rs @@ -453,8 +453,7 @@ pub fn command_to_string<'a>(command: impl Into>) -> String { for arg in command.argv.iter() { cmd.push(Cow::Borrowed(arg)); } - - shlex::join(cmd.iter().map(|e| e.as_ref())) + shlex::try_join(cmd.iter().map(|e| e.as_ref())).expect("Null byte unexpected") } impl WhatRanOutputWriter for SuperConsole { diff --git a/app/buck2_events/BUCK b/app/buck2_events/BUCK index f2b20c6f31e49..a98474575efab 100644 --- a/app/buck2_events/BUCK +++ b/app/buck2_events/BUCK @@ -31,7 +31,7 @@ rust_library( "//buck2/app/buck2_core:buck2_core", "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_error:buck2_error", - # @oss-disable: "//buck2/app/buck2_util:buck2_util", + "//buck2/app/buck2_util:buck2_util", "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/facebook/scribe_client:scribe_client", "//buck2/gazebo/dupe:dupe", diff --git a/app/buck2_events/Cargo.toml b/app/buck2_events/Cargo.toml index 3e08e58a5f38c..236066a82d53a 100644 --- a/app/buck2_events/Cargo.toml +++ b/app/buck2_events/Cargo.toml @@ -30,4 +30,5 @@ buck2_cli_proto = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } buck2_error = { workspace = true } +buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } diff --git a/app/buck2_events/src/dispatch.rs b/app/buck2_events/src/dispatch.rs index d4a908fa80f17..7050998c107f3 100644 --- a/app/buck2_events/src/dispatch.rs +++ b/app/buck2_events/src/dispatch.rs @@ -505,7 +505,7 @@ where } } - let previous_recorder = with_thread_local_recorder(|tl_recorder| std::mem::take(tl_recorder)); + let previous_recorder = with_thread_local_recorder(std::mem::take); let _guard = RestoreRecorder { previous_recorder }; f() } diff --git a/app/buck2_events/src/errors.rs b/app/buck2_events/src/errors.rs index 37a69ec10bc79..32449a9855da4 100644 --- a/app/buck2_events/src/errors.rs +++ b/app/buck2_events/src/errors.rs @@ -7,6 +7,8 @@ * of this source tree. */ +use gazebo::prelude::SliceExt; + pub fn create_error_report(err: &buck2_error::Error) -> buck2_data::ErrorReport { // Infra error by default if no category tag is set let category = err.get_category().map(|c| match c { @@ -29,6 +31,6 @@ pub fn create_error_report(err: &buck2_error::Error) -> buck2_data::ErrorReport message, telemetry_message, source_location, - tags: Vec::new(), + tags: err.tags().map(|t| *t as i32), } } diff --git a/app/buck2_events/src/metadata.rs b/app/buck2_events/src/metadata.rs index 2573f9f347a24..7c884418394e9 100644 --- a/app/buck2_events/src/metadata.rs +++ b/app/buck2_events/src/metadata.rs @@ -66,6 +66,16 @@ pub fn collect() -> HashMap { add_env_var(&mut map, "fbpackage_name", "FBPACKAGE_PACKAGE_NAME"); add_env_var(&mut map, "fbpackage_version", "FBPACKAGE_PACKAGE_VERSION"); add_env_var(&mut map, "fbpackage_release", "FBPACKAGE_PACKAGE_RELEASE"); + add_env_var( + &mut map, + "skycastle_workflow_run_id", + "SKYCASTLE_WORKFLOW_RUN_ID", + ); + add_env_var( + &mut map, + "skycastle_workflow_alias", + "SKYCASTLE_WORKFLOW_ALIAS", + ); map } diff --git a/app/buck2_events/src/sink.rs b/app/buck2_events/src/sink/mod.rs similarity index 93% rename from app/buck2_events/src/sink.rs rename to app/buck2_events/src/sink/mod.rs index 229f1f5479bfe..565ffefe826b7 100644 --- a/app/buck2_events/src/sink.rs +++ b/app/buck2_events/src/sink/mod.rs @@ -12,4 +12,5 @@ pub(crate) mod channel; pub(crate) mod null; pub mod scribe; +pub(crate) mod smart_truncate_event; pub mod tee; diff --git a/app/buck2_events/src/sink/scribe.rs b/app/buck2_events/src/sink/scribe.rs index dc5202723243d..32cffe2ae48f8 100644 --- a/app/buck2_events/src/sink/scribe.rs +++ b/app/buck2_events/src/sink/scribe.rs @@ -17,7 +17,6 @@ use fbinit::FacebookInit; #[cfg(fbcode_build)] mod fbcode { - use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; @@ -30,6 +29,7 @@ mod fbcode { use prost::Message; use crate::metadata; + use crate::sink::smart_truncate_event::smart_truncate_event; use crate::BuckEvent; use crate::Event; use crate::EventSink; @@ -103,7 +103,7 @@ mod fbcode { // Encodes message into something scribe understands. fn encode_message(mut event: BuckEvent, is_truncated: bool) -> Option> { - Self::smart_truncate_event(event.data_mut()); + smart_truncate_event(event.data_mut()); let proto: Box = event.into(); // Add a header byte to indicate this is _not_ base64 encoding. @@ -155,197 +155,6 @@ mod fbcode { Some(buf) } } - - fn smart_truncate_event(d: &mut buck2_data::buck_event::Data) { - use buck2_data::buck_event::Data; - - match d { - Data::SpanEnd(ref mut s) => { - use buck2_data::span_end_event::Data; - - match &mut s.data { - Some(Data::ActionExecution(ref mut action_execution)) => { - Self::truncate_action_execution_end(action_execution); - } - Some(Data::Command(ref mut command_end)) => { - Self::truncate_command_end(command_end, false); - } - Some(Data::TestEnd(ref mut test_end)) => { - Self::truncate_test_end(test_end); - } - _ => {} - }; - } - Data::Instant(ref mut inst) => { - use buck2_data::instant_event::Data; - match &mut inst.data { - Some(Data::TargetPatterns(ref mut target_patterns)) => { - Self::truncate_target_patterns(&mut target_patterns.target_patterns); - } - _ => {} - } - } - Data::Record(ref mut rec) => { - if let Some(buck2_data::record_event::Data::InvocationRecord( - ref mut invocation_record, - )) = rec.data - { - // FIXME(JakobDegen): The sum of the per-field limits adds up to more than the 1MB scribe limits - if let Some(ref mut file_watcher_stats) = - invocation_record.file_watcher_stats - { - Self::truncate_file_watcher_stats(file_watcher_stats); - } - if let Some(ref mut resolved_target_patterns) = - invocation_record.parsed_target_patterns - { - Self::truncate_target_patterns( - &mut resolved_target_patterns.target_patterns, - ); - // Clear `unresolved_traget_patterns` to save bandwidth. It has less information - // than `resolved` one does, and will never be used if `resolved` one is available. - if let Some(ref mut command_end) = invocation_record.command_end { - Self::truncate_command_end(command_end, true); - } - } else if let Some(ref mut command_end) = invocation_record.command_end { - Self::truncate_command_end(command_end, false); - } - - const MAX_CLI_ARGS_BYTES: usize = 512 * 1024; - let orig_len = invocation_record.cli_args.len(); - let mut bytes: usize = 0; - for (index, arg) in invocation_record.cli_args.iter().enumerate() { - bytes += arg.len(); - if bytes > MAX_CLI_ARGS_BYTES { - invocation_record.cli_args.truncate(index); - invocation_record.cli_args.push(format!( - "<>", - index, orig_len - )); - break; - } - } - - const MAX_ERROR_REPORT_BYTS: usize = 512 * 1024; - let max_per_report = - MAX_ERROR_REPORT_BYTS / invocation_record.errors.len().max(1); - for error in &mut invocation_record.errors { - error.message = truncate(&error.message, max_per_report / 2); - if let Some(telemetry_message) = &mut error.telemetry_message { - *telemetry_message = - truncate(telemetry_message, max_per_report / 2); - } - } - } - } - _ => {} - }; - } - - fn truncate_action_execution_end( - action_execution_end: &mut buck2_data::ActionExecutionEnd, - ) { - // truncate(...) can panic if asked to truncate too short. - const MIN_CMD_TRUNCATION: usize = 20; - let per_command_size_budget = - ((500 * 1024) / action_execution_end.commands.len().max(1)).max(MIN_CMD_TRUNCATION); - - let truncate_cmd = |cmd: &mut buck2_data::CommandExecution, truncate_all: bool| { - if let Some(details) = &mut cmd.details { - details.stderr = if truncate_all { - "<>".to_owned() - } else { - truncate(&details.stderr, per_command_size_budget) - }; - } - }; - - if let Some((last_command, retries)) = action_execution_end.commands.split_last_mut() { - for retried in retries { - truncate_cmd(retried, false); - } - // Current Scribe tailers don't read stderr of successful actions. - // Save some bytes. - truncate_cmd(last_command, !action_execution_end.failed); - } - } - - fn truncate_command_end( - command_end: &mut buck2_data::CommandEnd, - clear_target_patterns: bool, - ) { - use buck2_data::command_end::Data; - - if let Some(ref mut target_patterns) = match &mut command_end.data { - Some(Data::Build(build_command_end)) => { - Some(&mut build_command_end.unresolved_target_patterns) - } - Some(Data::Test(test_command_end)) => { - Some(&mut test_command_end.unresolved_target_patterns) - } - Some(Data::Install(install_command_end)) => { - Some(&mut install_command_end.unresolved_target_patterns) - } - Some(Data::Targets(targets_command_end)) => { - Some(&mut targets_command_end.unresolved_target_patterns) - } - _ => None, - } { - if clear_target_patterns { - target_patterns.clear(); - } else { - Self::truncate_target_patterns(target_patterns); - } - } - } - - fn truncate_file_watcher_stats(file_watcher_stats: &mut buck2_data::FileWatcherStats) { - const MAX_FILE_CHANGE_BYTES: usize = 100 * 1024; - let mut bytes: usize = 0; - for (index, ev) in file_watcher_stats.events.iter().enumerate() { - bytes += ev.path.len(); - if bytes > MAX_FILE_CHANGE_BYTES { - file_watcher_stats.events.truncate(index); - file_watcher_stats.incomplete_events_reason = Some(format!( - "Too long file change records ({} bytes, max {} bytes)", - bytes, MAX_FILE_CHANGE_BYTES - )); - break; - } - } - } - - fn truncate_test_end(test_end: &mut buck2_data::TestRunEnd) { - const MAX_TEST_NAMES_BYTES: usize = 512 * 1024; - if let Some(ref mut suite) = test_end.suite { - let orig_len = suite.test_names.len(); - let mut bytes: usize = 0; - for (index, test_name) in suite.test_names.iter().enumerate() { - bytes += test_name.len(); - if bytes > MAX_TEST_NAMES_BYTES { - suite.test_names.truncate(index); - let warn = format!("<>", index, orig_len); - suite.test_names.push(warn); - break; - } - } - } - } - - fn truncate_target_patterns(target_patterns: &mut Vec) { - const MAX_TARGET_PATTERNS_BYTES: usize = 512 * 1024; - let orig_len = target_patterns.len(); - let mut bytes: usize = 0; - for (index, target) in target_patterns.iter().enumerate() { - bytes += target.value.len(); - if bytes > MAX_TARGET_PATTERNS_BYTES { - target_patterns.truncate(index); - let warn = format!("<>", index, orig_len); - target_patterns.push(buck2_data::TargetPattern { value: warn }); - break; - } - } - } } impl EventSink for ThriftScribeSink { @@ -423,7 +232,7 @@ mod fbcode { Some(Data::RageResult(..)) => true, Some(Data::ReSession(..)) => true, Some(Data::StructuredError(..)) => true, - Some(Data::PersistSubprocess(..)) => true, + Some(Data::PersistEventLogSubprocess(..)) => true, None => false, _ => false, } @@ -439,444 +248,6 @@ mod fbcode { } } } - - #[cfg(test)] - mod tests { - use super::*; - - fn make_invocation_record( - data: buck2_data::InvocationRecord, - ) -> buck2_data::buck_event::Data { - buck2_data::buck_event::Data::Record(buck2_data::RecordEvent { - data: Some(buck2_data::record_event::Data::InvocationRecord(Box::new( - data, - ))), - }) - } - - fn make_action_execution_end( - data: buck2_data::ActionExecutionEnd, - ) -> buck2_data::buck_event::Data { - buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { - data: Some(buck2_data::span_end_event::Data::ActionExecution(Box::new( - data, - ))), - ..Default::default() - }) - } - - fn make_command_end(data: buck2_data::CommandEnd) -> buck2_data::buck_event::Data { - buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { - data: Some(buck2_data::span_end_event::Data::Command(data)), - ..Default::default() - }) - } - - fn make_build_command_end( - unresolved_target_patterns: Vec, - ) -> buck2_data::CommandEnd { - buck2_data::CommandEnd { - data: Some(buck2_data::command_end::Data::Build( - buck2_data::BuildCommandEnd { - unresolved_target_patterns, - }, - )), - ..Default::default() - } - } - - fn make_test_end(data: buck2_data::TestRunEnd) -> buck2_data::buck_event::Data { - buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { - data: Some(buck2_data::span_end_event::Data::TestEnd(data)), - ..Default::default() - }) - } - - fn make_command_execution_with_stderr(stderr: String) -> buck2_data::CommandExecution { - buck2_data::CommandExecution { - details: Some(buck2_data::CommandExecutionDetails { - stderr, - ..Default::default() - }), - ..Default::default() - } - } - - #[test] - fn smart_truncate_resolved_target_patterns_clears_unresolved_one() { - let mut record = buck2_data::InvocationRecord::default(); - let mut record_expected = record.clone(); - - let resolved_target_patterns = vec![buck2_data::TargetPattern { - value: "some_resolved_target".to_owned(), - }]; - record.parsed_target_patterns = Some(buck2_data::ParsedTargetPatterns { - target_patterns: resolved_target_patterns.clone(), - }); - // resolved_target_patterns is expected to be unchanged. - record_expected.parsed_target_patterns = Some(buck2_data::ParsedTargetPatterns { - target_patterns: resolved_target_patterns, - }); - - let unresolved_target_patterns = vec![buck2_data::TargetPattern { - value: "some_unresolved_target".to_owned(), - }]; - record.command_end = Some(make_build_command_end(unresolved_target_patterns)); - - // unresolved_target_patterns is expected to be empty. - record_expected.command_end = Some(make_build_command_end(vec![])); - - let mut event_data = make_invocation_record(record); - let event_data_expected = make_invocation_record(record_expected); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_unresolved_target_used_when_resolved_one_unavailable() { - let mut record = buck2_data::InvocationRecord::default(); - let mut record_expected = record.clone(); - - record.parsed_target_patterns = None; - record_expected.parsed_target_patterns = None; - - let unresolved_target_patterns = vec![buck2_data::TargetPattern { - value: "some_unresolved_target".to_owned(), - }]; - let command_end = make_build_command_end(unresolved_target_patterns); - - record.command_end = Some(command_end.clone()); - // unresolved_target_patterns is expected to be unchanged. - record_expected.command_end = Some(command_end); - - let mut event_data = make_invocation_record(record); - let event_data_expected = make_invocation_record(record_expected); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_action_execution_end_one_last_command_truncated() { - let command_execution_with_stderr = - make_command_execution_with_stderr("this is a test".to_owned()); - let command_execution_stderr_omitted = - make_command_execution_with_stderr("<>".to_owned()); - - let action_execution_end_with_stderrs = buck2_data::ActionExecutionEnd { - commands: vec![command_execution_with_stderr], - ..Default::default() - }; - let action_execution_end_last_stderr_omitted = buck2_data::ActionExecutionEnd { - commands: vec![command_execution_stderr_omitted], - ..Default::default() - }; - let mut event_data = make_action_execution_end(action_execution_end_with_stderrs); - let event_data_expected = - make_action_execution_end(action_execution_end_last_stderr_omitted); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_action_execution_end_long_stderr_command_truncated() { - let command_execution_with_stderr = - make_command_execution_with_stderr("this is a test".to_owned()); - let mut over_sized_str = "0123456789".repeat(10 * 1024); - over_sized_str.push_str("0123456789"); // 100k + 10; 10-byte over - let command_execution_with_long_stderr = - make_command_execution_with_stderr(over_sized_str); - let mut omitted_str = "0123456789".repeat(10 * 1024); - omitted_str.replace_range((50 * 1024 - 6)..(50 * 1024 + 6), "<>"); - let command_execution_stderr_partially_omitted = - make_command_execution_with_stderr(omitted_str); - let command_execution_stderr_all_omitted = - make_command_execution_with_stderr("<>".to_owned()); - - let action_execution_end_with_stderrs = buck2_data::ActionExecutionEnd { - commands: vec![ - command_execution_with_stderr.clone(), - command_execution_with_long_stderr.clone(), - command_execution_with_stderr.clone(), - command_execution_with_long_stderr, - command_execution_with_stderr.clone(), - ], - ..Default::default() - }; - let action_execution_end_last_stderr_omitted = buck2_data::ActionExecutionEnd { - commands: vec![ - command_execution_with_stderr.clone(), - command_execution_stderr_partially_omitted.clone(), - command_execution_with_stderr, - command_execution_stderr_partially_omitted, - command_execution_stderr_all_omitted, - ], - ..Default::default() - }; - let mut event_data = make_action_execution_end(action_execution_end_with_stderrs); - let event_data_expected = - make_action_execution_end(action_execution_end_last_stderr_omitted); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_build_command_end_short_target_patterns_not_truncated() { - let unresolved_target_patterns = vec![ - buck2_data::TargetPattern { - value: "hello".to_owned(), - }, - buck2_data::TargetPattern { - value: "world".to_owned(), - }, - buck2_data::TargetPattern { - value: "!\n".to_owned(), - }, - ]; - let command_end = make_build_command_end(unresolved_target_patterns); - - let mut event_data = make_command_end(command_end); - let event_data_expected = event_data.clone(); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_build_command_end_long_target_patterns_truncated() { - let unresolved_target_patterns = vec![ - buck2_data::TargetPattern { - value: "0123456789".repeat(20 * 1024), - }, - buck2_data::TargetPattern { - value: "0123456789".repeat(20 * 1024), - }, - buck2_data::TargetPattern { - value: "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over - }, - ]; - let command_end = make_build_command_end(unresolved_target_patterns); - - let unresolved_target_patterns_truncated = vec![ - buck2_data::TargetPattern { - value: "0123456789".repeat(20 * 1024), - }, - buck2_data::TargetPattern { - value: "0123456789".repeat(20 * 1024), - }, - buck2_data::TargetPattern { - value: "<>".to_owned(), - }, - ]; - let command_end_truncated = - make_build_command_end(unresolved_target_patterns_truncated); - - let mut event_data = make_command_end(command_end); - let event_data_expected = make_command_end(command_end_truncated); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_long_file_watcher_stats_truncated() { - let file_watcher_event = buck2_data::FileWatcherEvent { - path: "0123456789".repeat(3 * 1024), - ..Default::default() - }; - let file_watcher_stats = buck2_data::FileWatcherStats { - events: vec![ - file_watcher_event.clone(), - file_watcher_event.clone(), - file_watcher_event.clone(), - file_watcher_event.clone(), // 120k in total; 20k-byte over - ], - ..Default::default() - }; - let file_watcher_stats_truncated = buck2_data::FileWatcherStats { - events: vec![ - file_watcher_event.clone(), - file_watcher_event.clone(), - file_watcher_event, - ], - incomplete_events_reason: Some(format!( - "Too long file change records ({} bytes, max {} bytes)", - 120 * 1024, - 100 * 1024 - )), - ..Default::default() - }; - let record = buck2_data::InvocationRecord { - file_watcher_stats: Some(file_watcher_stats), - ..Default::default() - }; - let record_truncated = buck2_data::InvocationRecord { - file_watcher_stats: Some(file_watcher_stats_truncated), - ..Default::default() - }; - let mut event_data = make_invocation_record(record); - let event_data_expected = make_invocation_record(record_truncated); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_short_file_watcher_stats_not_truncated() { - let file_watcher_event = buck2_data::FileWatcherEvent { - path: "this is a test".to_owned(), - ..Default::default() - }; - let file_watcher_stats = buck2_data::FileWatcherStats { - events: vec![ - file_watcher_event.clone(), - file_watcher_event.clone(), - file_watcher_event, - ], - ..Default::default() - }; - let record = buck2_data::InvocationRecord { - file_watcher_stats: Some(file_watcher_stats), - ..Default::default() - }; - let mut event_data = make_invocation_record(record); - let event_data_expected = event_data.clone(); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_invocation_record_long_cli_args_truncated() { - let cli_args = vec![ - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over - ]; - let cli_args_truncated = vec![ - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), - "<>".to_owned(), - ]; - - let record = buck2_data::InvocationRecord { - cli_args, - ..Default::default() - }; - let record_truncated = buck2_data::InvocationRecord { - cli_args: cli_args_truncated, - ..Default::default() - }; - - let mut event_data = make_invocation_record(record); - let event_data_expected = make_invocation_record(record_truncated); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_invocation_record_short_cli_args_truncated() { - let cli_args = vec!["this is".to_owned(), "a test".to_owned()]; - - let record = buck2_data::InvocationRecord { - cli_args, - ..Default::default() - }; - - let mut event_data = make_invocation_record(record); - let event_data_expected = event_data.clone(); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - - #[test] - fn smart_truncate_invocation_record_error_reports_truncated() { - let errors = vec![ - buck2_data::ProcessedErrorReport { - message: "0123456789".repeat(200 * 1024), - telemetry_message: None, - ..Default::default() - }, - buck2_data::ProcessedErrorReport { - message: "0123456789".repeat(200 * 1024), - telemetry_message: Some("0123456789".repeat(200 * 1024)), - ..Default::default() - }, - ]; - - let mut event_data = make_invocation_record(buck2_data::InvocationRecord { - errors, - ..Default::default() - }); - ThriftScribeSink::smart_truncate_event(&mut event_data); - - let buck2_data::buck_event::Data::Record(record_event) = event_data else { - unreachable!() - }; - let Some(buck2_data::record_event::Data::InvocationRecord(invocation_record)) = - record_event.data - else { - unreachable!() - }; - let size = invocation_record - .errors - .into_iter() - .map(|e| e.message.len() + e.telemetry_message.map_or(0, |s| s.len())) - .sum::(); - assert!(size < 500 * 1024); - } - - #[test] - fn smart_truncate_test_end_long_test_names_truncated() { - let test_names = vec![ - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over - ]; - let test_names_truncated = vec![ - "0123456789".repeat(20 * 1024), - "0123456789".repeat(20 * 1024), - "<>".to_owned(), - ]; - - let test_end = buck2_data::TestRunEnd { - suite: Some(buck2_data::TestSuite { - test_names, - ..Default::default() - }), - ..Default::default() - }; - let test_end_truncated = buck2_data::TestRunEnd { - suite: Some(buck2_data::TestSuite { - test_names: test_names_truncated, - ..Default::default() - }), - ..Default::default() - }; - - let mut event_data = make_test_end(test_end); - let event_data_expected = make_test_end(test_end_truncated); - - ThriftScribeSink::smart_truncate_event(&mut event_data); - - assert_eq!(event_data, event_data_expected); - } - } } #[cfg(not(fbcode_build))] diff --git a/app/buck2_events/src/sink/smart_truncate_event.rs b/app/buck2_events/src/sink/smart_truncate_event.rs new file mode 100644 index 0000000000000..bbc874ee55f94 --- /dev/null +++ b/app/buck2_events/src/sink/smart_truncate_event.rs @@ -0,0 +1,625 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use buck2_util::truncate::truncate; + +#[cfg_attr(not(fbcode_build), allow(dead_code))] +pub(crate) fn smart_truncate_event(d: &mut buck2_data::buck_event::Data) { + use buck2_data::buck_event::Data; + + match d { + Data::SpanEnd(ref mut s) => { + use buck2_data::span_end_event::Data; + + match &mut s.data { + Some(Data::ActionExecution(ref mut action_execution)) => { + truncate_action_execution_end(action_execution); + } + Some(Data::Command(ref mut command_end)) => { + truncate_command_end(command_end, false); + } + Some(Data::TestEnd(ref mut test_end)) => { + truncate_test_end(test_end); + } + _ => {} + }; + } + Data::Instant(ref mut inst) => { + use buck2_data::instant_event::Data; + match &mut inst.data { + Some(Data::TargetPatterns(ref mut target_patterns)) => { + truncate_target_patterns(&mut target_patterns.target_patterns); + } + _ => {} + } + } + Data::Record(ref mut rec) => { + if let Some(buck2_data::record_event::Data::InvocationRecord(invocation_record)) = + &mut rec.data + { + truncate_invocation_record(invocation_record) + } + } + _ => {} + }; +} + +fn truncate_invocation_record(invocation_record: &mut buck2_data::InvocationRecord) { + // FIXME(JakobDegen): The sum of the per-field limits adds up to more than the 1MB scribe limits + if let Some(ref mut file_watcher_stats) = invocation_record.file_watcher_stats { + truncate_file_watcher_stats(file_watcher_stats); + } + if let Some(ref mut resolved_target_patterns) = invocation_record.parsed_target_patterns { + truncate_target_patterns(&mut resolved_target_patterns.target_patterns); + // Clear `unresolved_traget_patterns` to save bandwidth. It has less information + // than `resolved` one does, and will never be used if `resolved` one is available. + if let Some(ref mut command_end) = invocation_record.command_end { + truncate_command_end(command_end, true); + } + } else if let Some(ref mut command_end) = invocation_record.command_end { + truncate_command_end(command_end, false); + } + + const MAX_CLI_ARGS_BYTES: usize = 512 * 1024; + let orig_len = invocation_record.cli_args.len(); + let mut bytes: usize = 0; + for (index, arg) in invocation_record.cli_args.iter().enumerate() { + bytes += arg.len(); + if bytes > MAX_CLI_ARGS_BYTES { + invocation_record.cli_args.truncate(index); + invocation_record + .cli_args + .push(format!("<>", index, orig_len)); + break; + } + } + + const MAX_ERROR_REPORT_BYTS: usize = 512 * 1024; + let max_per_report = MAX_ERROR_REPORT_BYTS / invocation_record.errors.len().max(1); + for error in &mut invocation_record.errors { + error.message = truncate(&error.message, max_per_report / 2); + if let Some(telemetry_message) = &mut error.telemetry_message { + *telemetry_message = truncate(telemetry_message, max_per_report / 2); + } + } +} + +fn truncate_action_execution_end(action_execution_end: &mut buck2_data::ActionExecutionEnd) { + // truncate(...) can panic if asked to truncate too short. + const MIN_CMD_TRUNCATION: usize = 20; + let per_command_size_budget = + ((500 * 1024) / action_execution_end.commands.len().max(1)).max(MIN_CMD_TRUNCATION); + + let truncate_cmd = |cmd: &mut buck2_data::CommandExecution, truncate_all: bool| { + if let Some(details) = &mut cmd.details { + details.stderr = if truncate_all { + "<>".to_owned() + } else { + truncate(&details.stderr, per_command_size_budget) + }; + } + }; + + if let Some((last_command, retries)) = action_execution_end.commands.split_last_mut() { + for retried in retries { + truncate_cmd(retried, false); + } + // Current Scribe tailers don't read stderr of successful actions. + // Save some bytes. + truncate_cmd(last_command, !action_execution_end.failed); + } +} + +fn truncate_command_end(command_end: &mut buck2_data::CommandEnd, clear_target_patterns: bool) { + use buck2_data::command_end::Data; + + if let Some(ref mut target_patterns) = match &mut command_end.data { + Some(Data::Build(build_command_end)) => { + Some(&mut build_command_end.unresolved_target_patterns) + } + Some(Data::Test(test_command_end)) => { + Some(&mut test_command_end.unresolved_target_patterns) + } + Some(Data::Install(install_command_end)) => { + Some(&mut install_command_end.unresolved_target_patterns) + } + Some(Data::Targets(targets_command_end)) => { + Some(&mut targets_command_end.unresolved_target_patterns) + } + _ => None, + } { + if clear_target_patterns { + target_patterns.clear(); + } else { + truncate_target_patterns(target_patterns); + } + } +} + +fn truncate_file_watcher_stats(file_watcher_stats: &mut buck2_data::FileWatcherStats) { + const MAX_FILE_CHANGE_BYTES: usize = 100 * 1024; + let mut bytes: usize = 0; + for (index, ev) in file_watcher_stats.events.iter().enumerate() { + bytes += ev.path.len(); + if bytes > MAX_FILE_CHANGE_BYTES { + file_watcher_stats.events.truncate(index); + file_watcher_stats.incomplete_events_reason = Some(format!( + "Too long file change records ({} bytes, max {} bytes)", + bytes, MAX_FILE_CHANGE_BYTES + )); + break; + } + } +} + +fn truncate_test_end(test_end: &mut buck2_data::TestRunEnd) { + const MAX_TEST_NAMES_BYTES: usize = 512 * 1024; + if let Some(ref mut suite) = test_end.suite { + let orig_len = suite.test_names.len(); + let mut bytes: usize = 0; + for (index, test_name) in suite.test_names.iter().enumerate() { + bytes += test_name.len(); + if bytes > MAX_TEST_NAMES_BYTES { + suite.test_names.truncate(index); + let warn = format!("<>", index, orig_len); + suite.test_names.push(warn); + break; + } + } + } +} + +fn truncate_target_patterns(target_patterns: &mut Vec) { + const MAX_TARGET_PATTERNS_BYTES: usize = 512 * 1024; + let orig_len = target_patterns.len(); + let mut bytes: usize = 0; + for (index, target) in target_patterns.iter().enumerate() { + bytes += target.value.len(); + if bytes > MAX_TARGET_PATTERNS_BYTES { + target_patterns.truncate(index); + let warn = format!("<>", index, orig_len); + target_patterns.push(buck2_data::TargetPattern { value: warn }); + break; + } + } +} + +#[cfg(test)] +mod tests { + use crate::sink::smart_truncate_event::smart_truncate_event; + + fn make_invocation_record(data: buck2_data::InvocationRecord) -> buck2_data::buck_event::Data { + buck2_data::buck_event::Data::Record(buck2_data::RecordEvent { + data: Some(buck2_data::record_event::Data::InvocationRecord(Box::new( + data, + ))), + }) + } + + fn make_action_execution_end( + data: buck2_data::ActionExecutionEnd, + ) -> buck2_data::buck_event::Data { + buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { + data: Some(buck2_data::span_end_event::Data::ActionExecution(Box::new( + data, + ))), + ..Default::default() + }) + } + + fn make_command_end(data: buck2_data::CommandEnd) -> buck2_data::buck_event::Data { + buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { + data: Some(buck2_data::span_end_event::Data::Command(data)), + ..Default::default() + }) + } + + fn make_build_command_end( + unresolved_target_patterns: Vec, + ) -> buck2_data::CommandEnd { + buck2_data::CommandEnd { + data: Some(buck2_data::command_end::Data::Build( + buck2_data::BuildCommandEnd { + unresolved_target_patterns, + }, + )), + ..Default::default() + } + } + + fn make_test_end(data: buck2_data::TestRunEnd) -> buck2_data::buck_event::Data { + buck2_data::buck_event::Data::SpanEnd(buck2_data::SpanEndEvent { + data: Some(buck2_data::span_end_event::Data::TestEnd(data)), + ..Default::default() + }) + } + + fn make_command_execution_with_stderr(stderr: String) -> buck2_data::CommandExecution { + buck2_data::CommandExecution { + details: Some(buck2_data::CommandExecutionDetails { + stderr, + ..Default::default() + }), + ..Default::default() + } + } + + #[test] + fn smart_truncate_resolved_target_patterns_clears_unresolved_one() { + let mut record = buck2_data::InvocationRecord::default(); + let mut record_expected = record.clone(); + + let resolved_target_patterns = vec![buck2_data::TargetPattern { + value: "some_resolved_target".to_owned(), + }]; + record.parsed_target_patterns = Some(buck2_data::ParsedTargetPatterns { + target_patterns: resolved_target_patterns.clone(), + }); + // resolved_target_patterns is expected to be unchanged. + record_expected.parsed_target_patterns = Some(buck2_data::ParsedTargetPatterns { + target_patterns: resolved_target_patterns, + }); + + let unresolved_target_patterns = vec![buck2_data::TargetPattern { + value: "some_unresolved_target".to_owned(), + }]; + record.command_end = Some(make_build_command_end(unresolved_target_patterns)); + + // unresolved_target_patterns is expected to be empty. + record_expected.command_end = Some(make_build_command_end(vec![])); + + let mut event_data = make_invocation_record(record); + let event_data_expected = make_invocation_record(record_expected); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_unresolved_target_used_when_resolved_one_unavailable() { + let mut record = buck2_data::InvocationRecord::default(); + let mut record_expected = record.clone(); + + record.parsed_target_patterns = None; + record_expected.parsed_target_patterns = None; + + let unresolved_target_patterns = vec![buck2_data::TargetPattern { + value: "some_unresolved_target".to_owned(), + }]; + let command_end = make_build_command_end(unresolved_target_patterns); + + record.command_end = Some(command_end.clone()); + // unresolved_target_patterns is expected to be unchanged. + record_expected.command_end = Some(command_end); + + let mut event_data = make_invocation_record(record); + let event_data_expected = make_invocation_record(record_expected); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_action_execution_end_one_last_command_truncated() { + let command_execution_with_stderr = + make_command_execution_with_stderr("this is a test".to_owned()); + let command_execution_stderr_omitted = + make_command_execution_with_stderr("<>".to_owned()); + + let action_execution_end_with_stderrs = buck2_data::ActionExecutionEnd { + commands: vec![command_execution_with_stderr], + ..Default::default() + }; + let action_execution_end_last_stderr_omitted = buck2_data::ActionExecutionEnd { + commands: vec![command_execution_stderr_omitted], + ..Default::default() + }; + let mut event_data = make_action_execution_end(action_execution_end_with_stderrs); + let event_data_expected = + make_action_execution_end(action_execution_end_last_stderr_omitted); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_action_execution_end_long_stderr_command_truncated() { + let command_execution_with_stderr = + make_command_execution_with_stderr("this is a test".to_owned()); + let mut over_sized_str = "0123456789".repeat(10 * 1024); + over_sized_str.push_str("0123456789"); // 100k + 10; 10-byte over + let command_execution_with_long_stderr = make_command_execution_with_stderr(over_sized_str); + let mut omitted_str = "0123456789".repeat(10 * 1024); + omitted_str.replace_range((50 * 1024 - 6)..(50 * 1024 + 6), "<>"); + let command_execution_stderr_partially_omitted = + make_command_execution_with_stderr(omitted_str); + let command_execution_stderr_all_omitted = + make_command_execution_with_stderr("<>".to_owned()); + + let action_execution_end_with_stderrs = buck2_data::ActionExecutionEnd { + commands: vec![ + command_execution_with_stderr.clone(), + command_execution_with_long_stderr.clone(), + command_execution_with_stderr.clone(), + command_execution_with_long_stderr, + command_execution_with_stderr.clone(), + ], + ..Default::default() + }; + let action_execution_end_last_stderr_omitted = buck2_data::ActionExecutionEnd { + commands: vec![ + command_execution_with_stderr.clone(), + command_execution_stderr_partially_omitted.clone(), + command_execution_with_stderr, + command_execution_stderr_partially_omitted, + command_execution_stderr_all_omitted, + ], + ..Default::default() + }; + let mut event_data = make_action_execution_end(action_execution_end_with_stderrs); + let event_data_expected = + make_action_execution_end(action_execution_end_last_stderr_omitted); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_build_command_end_short_target_patterns_not_truncated() { + let unresolved_target_patterns = vec![ + buck2_data::TargetPattern { + value: "hello".to_owned(), + }, + buck2_data::TargetPattern { + value: "world".to_owned(), + }, + buck2_data::TargetPattern { + value: "!\n".to_owned(), + }, + ]; + let command_end = make_build_command_end(unresolved_target_patterns); + + let mut event_data = make_command_end(command_end); + let event_data_expected = event_data.clone(); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_build_command_end_long_target_patterns_truncated() { + let unresolved_target_patterns = vec![ + buck2_data::TargetPattern { + value: "0123456789".repeat(20 * 1024), + }, + buck2_data::TargetPattern { + value: "0123456789".repeat(20 * 1024), + }, + buck2_data::TargetPattern { + value: "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over + }, + ]; + let command_end = make_build_command_end(unresolved_target_patterns); + + let unresolved_target_patterns_truncated = vec![ + buck2_data::TargetPattern { + value: "0123456789".repeat(20 * 1024), + }, + buck2_data::TargetPattern { + value: "0123456789".repeat(20 * 1024), + }, + buck2_data::TargetPattern { + value: "<>".to_owned(), + }, + ]; + let command_end_truncated = make_build_command_end(unresolved_target_patterns_truncated); + + let mut event_data = make_command_end(command_end); + let event_data_expected = make_command_end(command_end_truncated); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_long_file_watcher_stats_truncated() { + let file_watcher_event = buck2_data::FileWatcherEvent { + path: "0123456789".repeat(3 * 1024), + ..Default::default() + }; + let file_watcher_stats = buck2_data::FileWatcherStats { + events: vec![ + file_watcher_event.clone(), + file_watcher_event.clone(), + file_watcher_event.clone(), + file_watcher_event.clone(), // 120k in total; 20k-byte over + ], + ..Default::default() + }; + let file_watcher_stats_truncated = buck2_data::FileWatcherStats { + events: vec![ + file_watcher_event.clone(), + file_watcher_event.clone(), + file_watcher_event, + ], + incomplete_events_reason: Some(format!( + "Too long file change records ({} bytes, max {} bytes)", + 120 * 1024, + 100 * 1024 + )), + ..Default::default() + }; + let record = buck2_data::InvocationRecord { + file_watcher_stats: Some(file_watcher_stats), + ..Default::default() + }; + let record_truncated = buck2_data::InvocationRecord { + file_watcher_stats: Some(file_watcher_stats_truncated), + ..Default::default() + }; + let mut event_data = make_invocation_record(record); + let event_data_expected = make_invocation_record(record_truncated); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_short_file_watcher_stats_not_truncated() { + let file_watcher_event = buck2_data::FileWatcherEvent { + path: "this is a test".to_owned(), + ..Default::default() + }; + let file_watcher_stats = buck2_data::FileWatcherStats { + events: vec![ + file_watcher_event.clone(), + file_watcher_event.clone(), + file_watcher_event, + ], + ..Default::default() + }; + let record = buck2_data::InvocationRecord { + file_watcher_stats: Some(file_watcher_stats), + ..Default::default() + }; + let mut event_data = make_invocation_record(record); + let event_data_expected = event_data.clone(); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_invocation_record_long_cli_args_truncated() { + let cli_args = vec![ + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over + ]; + let cli_args_truncated = vec![ + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), + "<>".to_owned(), + ]; + + let record = buck2_data::InvocationRecord { + cli_args, + ..Default::default() + }; + let record_truncated = buck2_data::InvocationRecord { + cli_args: cli_args_truncated, + ..Default::default() + }; + + let mut event_data = make_invocation_record(record); + let event_data_expected = make_invocation_record(record_truncated); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_invocation_record_short_cli_args_truncated() { + let cli_args = vec!["this is".to_owned(), "a test".to_owned()]; + + let record = buck2_data::InvocationRecord { + cli_args, + ..Default::default() + }; + + let mut event_data = make_invocation_record(record); + let event_data_expected = event_data.clone(); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } + + #[test] + fn smart_truncate_invocation_record_error_reports_truncated() { + let errors = vec![ + buck2_data::ProcessedErrorReport { + message: "0123456789".repeat(200 * 1024), + telemetry_message: None, + ..Default::default() + }, + buck2_data::ProcessedErrorReport { + message: "0123456789".repeat(200 * 1024), + telemetry_message: Some("0123456789".repeat(200 * 1024)), + ..Default::default() + }, + ]; + + let mut event_data = make_invocation_record(buck2_data::InvocationRecord { + errors, + ..Default::default() + }); + smart_truncate_event(&mut event_data); + + let buck2_data::buck_event::Data::Record(record_event) = event_data else { + unreachable!() + }; + let Some(buck2_data::record_event::Data::InvocationRecord(invocation_record)) = + record_event.data + else { + unreachable!() + }; + let size = invocation_record + .errors + .into_iter() + .map(|e| e.message.len() + e.telemetry_message.map_or(0, |s| s.len())) + .sum::(); + assert!(size < 500 * 1024); + } + + #[test] + fn smart_truncate_test_end_long_test_names_truncated() { + let test_names = vec![ + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), // 600k in total; 88k-byte over + ]; + let test_names_truncated = vec![ + "0123456789".repeat(20 * 1024), + "0123456789".repeat(20 * 1024), + "<>".to_owned(), + ]; + + let test_end = buck2_data::TestRunEnd { + suite: Some(buck2_data::TestSuite { + test_names, + ..Default::default() + }), + ..Default::default() + }; + let test_end_truncated = buck2_data::TestRunEnd { + suite: Some(buck2_data::TestSuite { + test_names: test_names_truncated, + ..Default::default() + }), + ..Default::default() + }; + + let mut event_data = make_test_end(test_end); + let event_data_expected = make_test_end(test_end_truncated); + + smart_truncate_event(&mut event_data); + + assert_eq!(event_data, event_data_expected); + } +} diff --git a/app/buck2_execute/BUCK b/app/buck2_execute/BUCK index 3432be24c37c9..a1431ed79eae5 100644 --- a/app/buck2_execute/BUCK +++ b/app/buck2_execute/BUCK @@ -17,6 +17,7 @@ rust_library( ], deps = [ "fbsource//third-party/rust:anyhow", + "fbsource//third-party/rust:async-recursion", "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:bytes", "fbsource//third-party/rust:chrono", @@ -33,6 +34,7 @@ rust_library( "fbsource//third-party/rust:itertools", "fbsource//third-party/rust:num_cpus", "fbsource//third-party/rust:once_cell", + "fbsource//third-party/rust:pathdiff", "fbsource//third-party/rust:prost", "fbsource//third-party/rust:ref-cast", "fbsource//third-party/rust:serde", @@ -46,6 +48,7 @@ rust_library( "fbsource//third-party/rust:tracing", "//buck2/allocative/allocative:allocative", "//buck2/app/buck2_action_metadata_proto:buck2_action_metadata_proto", + "//buck2/app/buck2_build_info:buck2_build_info", "//buck2/app/buck2_cli_proto:buck2_cli_proto", "//buck2/app/buck2_common:buck2_common", "//buck2/app/buck2_core:buck2_core", diff --git a/app/buck2_execute/Cargo.toml b/app/buck2_execute/Cargo.toml index 38e18383044d5..88f8f5f49592b 100644 --- a/app/buck2_execute/Cargo.toml +++ b/app/buck2_execute/Cargo.toml @@ -8,6 +8,7 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } +async-recursion = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } @@ -24,6 +25,7 @@ indexmap = { workspace = true } itertools = { workspace = true } num_cpus = { workspace = true } once_cell = { workspace = true } +pathdiff = { workspace = true } prost = { workspace = true } ref-cast = { workspace = true } serde = { workspace = true } @@ -45,6 +47,7 @@ sorted_vector_map = { workspace = true } starlark_map = { workspace = true } buck2_action_metadata_proto = { workspace = true } +buck2_build_info = { workspace = true } buck2_cli_proto = { workspace = true } buck2_common = { workspace = true } buck2_core = { workspace = true } @@ -55,6 +58,7 @@ buck2_futures = { workspace = true } buck2_http = { workspace = true } buck2_miniperf_proto = { workspace = true } buck2_re_configuration = { workspace = true } +buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } [dev-dependencies] diff --git a/app/buck2_execute/src/entry.rs b/app/buck2_execute/src/entry.rs index 64a592de2e104..d9c80973068da 100644 --- a/app/buck2_execute/src/entry.rs +++ b/app/buck2_execute/src/entry.rs @@ -7,10 +7,12 @@ * of this source tree. */ +use std::ops::Add; use std::time::Duration; use std::time::Instant; use anyhow::Context as _; +use async_recursion::async_recursion; use buck2_common::file_ops::FileDigest; use buck2_common::file_ops::FileDigestConfig; use buck2_common::file_ops::FileMetadata; @@ -20,21 +22,41 @@ use buck2_core::directory::DirectoryEntry; use buck2_core::fs::fs_util; use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; use buck2_core::fs::paths::file_name::FileNameBuf; +use buck2_core::fs::paths::RelativePath; +use derive_more::Add; use faccess::PathExt; +use futures::future::try_join; +use futures::future::try_join_all; +use futures::Future; +use once_cell::sync::Lazy; +use pathdiff::diff_paths; +use tokio::sync::Semaphore; use crate::directory::new_symlink; use crate::directory::ActionDirectoryBuilder; use crate::directory::ActionDirectoryEntry; use crate::directory::ActionDirectoryMember; +use crate::execute::blocking::BlockingExecutor; +#[derive(Add, Default)] pub struct HashingInfo { pub hashing_duration: Duration, pub hashed_artifacts_count: u64, } -pub fn build_entry_from_disk( - mut path: AbsNormPathBuf, +impl HashingInfo { + fn new(hashing_duration: Duration, hashed_artifacts_count: u64) -> HashingInfo { + HashingInfo { + hashing_duration, + hashed_artifacts_count, + } + } +} + +pub async fn build_entry_from_disk( + path: AbsNormPathBuf, digest_config: FileDigestConfig, + blocking_executor: &dyn BlockingExecutor, ) -> anyhow::Result<( Option>, HashingInfo, @@ -43,34 +65,23 @@ pub fn build_entry_from_disk( let m = match std::fs::symlink_metadata(&path) { Ok(m) => m, Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => { - return Ok(( - None, - HashingInfo { - hashing_duration: Duration::ZERO, - hashed_artifacts_count: 0, - }, - )); + return Ok((None, HashingInfo::default())); } Err(err) => return Err(err.into()), }; - let hashing_start = Instant::now(); - let mut hashed_artifacts_count = 0; + let mut hashing_info = HashingInfo::default(); let value = match FileType::from(m.file_type()) { FileType::File => { - hashed_artifacts_count += 1; - DirectoryEntry::Leaf(ActionDirectoryMember::File(FileMetadata { - digest: TrackedFileDigest::new( - FileDigest::from_file(&path, digest_config)?, - digest_config.as_cas_digest_config(), - ), - is_executable: path.executable(), - })) + let (file_metadata, file_hashing_info): (FileMetadata, HashingInfo) = + build_file_metadata(path, digest_config, blocking_executor).await?; + hashing_info = hashing_info.add(file_hashing_info); + DirectoryEntry::Leaf(ActionDirectoryMember::File(file_metadata)) } - FileType::Symlink => DirectoryEntry::Leaf(new_symlink(fs_util::read_link(&path)?)?), - + FileType::Symlink => DirectoryEntry::Leaf(create_symlink(&path)?), FileType::Directory => { - let (dir, count) = build_dir_from_disk(&mut path, digest_config)?; - hashed_artifacts_count += count; + let (dir, dir_hashing_info) = + build_dir_from_disk(path, digest_config, blocking_executor).await?; + hashing_info = hashing_info.add(dir_hashing_info); DirectoryEntry::Dir(dir) } FileType::Unknown => { @@ -80,23 +91,28 @@ pub fn build_entry_from_disk( )); } }; - let hashing_duration = hashing_start.elapsed(); - Ok(( - Some(value), - HashingInfo { - hashing_duration, - hashed_artifacts_count, - }, - )) + + Ok((Some(value), hashing_info)) } -fn build_dir_from_disk( - disk_path: &mut AbsNormPathBuf, +#[async_recursion] +async fn build_dir_from_disk( + disk_path: AbsNormPathBuf, digest_config: FileDigestConfig, -) -> anyhow::Result<(ActionDirectoryBuilder, u64)> { + blocking_executor: &dyn BlockingExecutor, +) -> anyhow::Result<(ActionDirectoryBuilder, HashingInfo)> { let mut builder = ActionDirectoryBuilder::empty(); - let mut hashed_artifacts_count = 0; - for file in fs_util::read_dir(&disk_path)? { + let mut hashing_info = HashingInfo::default(); + + let mut directory_names: Vec = Vec::new(); + let mut directory_futures: Vec<_> = Vec::new(); + let mut file_names: Vec = Vec::new(); + let mut file_futures: Vec<_> = Vec::new(); + + let files = blocking_executor + .execute_io_inline(|| fs_util::read_dir(&disk_path)) + .await?; + for file in files { let file = file?; let filetype = file.file_type()?; let filename = file.file_name(); @@ -105,40 +121,98 @@ fn build_dir_from_disk( .to_str() .context("Filename is not UTF-8") .and_then(|f| FileNameBuf::try_from(f.to_owned())) - .with_context(|| format!("Invalid filename: {}", disk_path.display()))?; + .with_context(|| format!("Invalid filename: {}", disk_path.clone().display()))?; + + let mut child_disk_path = disk_path.clone(); + child_disk_path.push(&filename); - disk_path.push(&filename); match FileType::from(filetype) { FileType::File => { - let metadata = FileMetadata { - digest: TrackedFileDigest::new( - FileDigest::from_file(disk_path, digest_config)?, - digest_config.as_cas_digest_config(), - ), - is_executable: file.path().executable(), - }; - builder.insert( - filename, - DirectoryEntry::Leaf(ActionDirectoryMember::File(metadata)), - )?; - hashed_artifacts_count += 1; + let file_future = + build_file_metadata(child_disk_path, digest_config, blocking_executor); + file_names.push(filename); + file_futures.push(file_future) } FileType::Symlink => { builder.insert( filename, - DirectoryEntry::Leaf(new_symlink(fs_util::read_link(&disk_path)?)?), + DirectoryEntry::Leaf(create_symlink(&child_disk_path)?), )?; } FileType::Directory => { - let (dir, hashed_files) = build_dir_from_disk(disk_path, digest_config)?; - builder.insert(filename, DirectoryEntry::Dir(dir))?; - hashed_artifacts_count += hashed_files; + let dir_future = + build_dir_from_disk(child_disk_path, digest_config, blocking_executor); + directory_names.push(filename); + directory_futures.push(dir_future); } FileType::Unknown => (), }; + } - disk_path.pop(); + let (file_results, dir_results) = + try_join(try_join_all(file_futures), try_join_all(directory_futures)).await?; + + for (filename, file_res) in file_names.into_iter().zip(file_results.into_iter()) { + let (file_metadata, file_hashing_info) = file_res; + hashing_info = hashing_info.add(file_hashing_info); + builder.insert( + filename, + DirectoryEntry::Leaf(ActionDirectoryMember::File(file_metadata)), + )?; } - Ok((builder, hashed_artifacts_count)) + for (filename, dir_res) in directory_names.into_iter().zip(dir_results.into_iter()) { + let (dir_builder, dir_hashing_info) = dir_res; + hashing_info = hashing_info.add(dir_hashing_info); + builder.insert(filename, DirectoryEntry::Dir(dir_builder))?; + } + + Ok((builder, hashing_info)) +} + +fn build_file_metadata( + disk_path: AbsNormPathBuf, + digest_config: FileDigestConfig, + blocking_executor: &dyn BlockingExecutor, +) -> impl Future> + '_ { + static SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(100)); + let exec_path = disk_path.clone(); + let executable = blocking_executor.execute_io_inline(move || Ok(exec_path.executable())); + let file_digest = + tokio::task::spawn_blocking(move || FileDigest::from_file(&disk_path, digest_config)); + + async move { + let _permit = SEMAPHORE.acquire().await.unwrap(); + let hashing_start = Instant::now(); + let file_digest = file_digest.await??; + let hashing_duration = HashingInfo::new(hashing_start.elapsed(), 1); + let file_metadata = FileMetadata { + digest: TrackedFileDigest::new(file_digest, digest_config.as_cas_digest_config()), + is_executable: executable.await?, + }; + + Ok((file_metadata, hashing_duration)) + } +} + +fn create_symlink(path: &AbsNormPathBuf) -> anyhow::Result { + let mut symlink_target = fs_util::read_link(path)?; + if cfg!(windows) && symlink_target.is_relative() { + let directory_path = path + .parent() + .context(format!("failed to get parent of {}", path.display()))?; + let canonical_path = fs_util::canonicalize(directory_path).context(format!( + "failed to get canonical path of {}", + directory_path.display() + ))?; + let normalized_target = symlink_target + .to_str() + .context("can't convert path to str")? + .replace('\\', "/"); + let target_abspath = + canonical_path.join_normalized(RelativePath::from_path(&normalized_target)?)?; + symlink_target = + diff_paths(target_abspath, directory_path).context("can't calculate relative path")?; + } + new_symlink(symlink_target) } diff --git a/app/buck2_execute/src/execute/blocking.rs b/app/buck2_execute/src/execute/blocking.rs index 6d6354363dc22..a65a34bb70ff3 100644 --- a/app/buck2_execute/src/execute/blocking.rs +++ b/app/buck2_execute/src/execute/blocking.rs @@ -15,6 +15,7 @@ use async_trait::async_trait; use buck2_core::buck2_env; use buck2_core::fs::project::ProjectRoot; use buck2_futures::cancellation::CancellationContext; +use buck2_util::threads::thread_spawn; use crossbeam_channel::unbounded; use dice::DiceComputations; use dice::UserComputationData; @@ -103,15 +104,13 @@ impl BuckBlockingExecutor { for i in 0..io_threads { let command_receiver = command_receiver.clone(); let fs = fs.dupe(); - std::thread::Builder::new() - .name(format!("buck-io-{}", i)) - .spawn(move || { - for ThreadPoolIoRequest { sender, io } in command_receiver.iter() { - let res = io.execute(&fs); - let _ignored = sender.send(res); - } - }) - .context("Failed to spawn io worker")?; + thread_spawn(&format!("buck-io-{}", i), move || { + for ThreadPoolIoRequest { sender, io } in command_receiver.iter() { + let res = io.execute(&fs); + let _ignored = sender.send(res); + } + }) + .context("Failed to spawn io worker")?; } Ok(Self { diff --git a/app/buck2_execute/src/execute/result.rs b/app/buck2_execute/src/execute/result.rs index 9285ff5df5085..55e7e28bc7fb7 100644 --- a/app/buck2_execute/src/execute/result.rs +++ b/app/buck2_execute/src/execute/result.rs @@ -119,6 +119,9 @@ pub struct CommandExecutionMetadata { /// How many artifacts we hashed pub hashed_artifacts_count: u64, + + /// How long this command spent waiting to run + pub queue_duration: Option, } impl CommandExecutionMetadata { @@ -136,6 +139,7 @@ impl CommandExecutionMetadata { execution_stats: metadata.execution_stats, hashing_duration: metadata.hashing_duration.try_into().ok(), hashed_artifacts_count: metadata.hashed_artifacts_count.try_into().ok().unwrap_or(0), + queue_duration: metadata.queue_duration.and_then(|d| d.try_into().ok()), } } } @@ -150,6 +154,7 @@ impl Default for CommandExecutionMetadata { input_materialization_duration: Duration::default(), hashing_duration: Duration::default(), hashed_artifacts_count: 0, + queue_duration: None, } } } @@ -366,6 +371,7 @@ mod tests { input_materialization_duration: Duration::from_secs(6), hashing_duration: Duration::from_secs(7), hashed_artifacts_count: 8, + queue_duration: Some(Duration::from_secs(9)), }; let std_streams = CommandStdStreams::Local { stdout: [65, 66, 67].to_vec(), // ABC @@ -435,6 +441,10 @@ mod tests { nanos: 0, }), hashed_artifacts_count: 8, + queue_duration: Some(Duration { + seconds: 9, + nanos: 0, + }), }; let command_execution_details = buck2_data::CommandExecutionDetails { signed_exit_code: Some(456), diff --git a/app/buck2_execute/src/materialize/http.rs b/app/buck2_execute/src/materialize/http.rs index b5f6121fbcef7..ff9cb63d11fd2 100644 --- a/app/buck2_execute/src/materialize/http.rs +++ b/app/buck2_execute/src/materialize/http.rs @@ -354,7 +354,7 @@ async fn copy_and_hash( } #[cfg(test)] -mod test { +mod tests { use assert_matches::assert_matches; use buck2_common::cas_digest::testing; use futures::stream; diff --git a/app/buck2_execute/src/re/client.rs b/app/buck2_execute/src/re/client.rs index 9c2da806337c2..4f32d5269bdfb 100644 --- a/app/buck2_execute/src/re/client.rs +++ b/app/buck2_execute/src/re/client.rs @@ -969,6 +969,9 @@ impl RemoteExecutionClientImpl { platform: Some(re_platform(platform)), do_not_cache: skip_cache_write, buck_info: Some(BuckInfo { + version: buck2_build_info::revision() + .map(|s| s.to_owned()) + .unwrap_or_default(), build_id: identity.trace_id.to_string(), ..Default::default() }), diff --git a/app/buck2_execute/src/re/remote_action_result.rs b/app/buck2_execute/src/re/remote_action_result.rs index 65646488be8c6..35d318e7daf33 100644 --- a/app/buck2_execute/src/re/remote_action_result.rs +++ b/app/buck2_execute/src/re/remote_action_result.rs @@ -172,8 +172,10 @@ impl RemoteActionResult for ActionResultResponse { fn timing(&self) -> CommandExecutionMetadata { let mut timing = timing_from_re_metadata(&self.action_result.execution_metadata); - timing.wall_time = Duration::ZERO; // This was a cache hit so we didn't wait. - timing.input_materialization_duration = Duration::ZERO; // This was a cache hit so we didn't wait. + // This was a cache hit so we didn't wait at all + timing.wall_time = Duration::ZERO; + timing.input_materialization_duration = Duration::ZERO; + timing.queue_duration = None; timing } @@ -252,6 +254,10 @@ fn timing_from_re_metadata(meta: &TExecutedActionMetadata) -> CommandExecutionMe .input_fetch_completed_timestamp .saturating_duration_since(&meta.input_fetch_start_timestamp); + let queue_duration = meta + .worker_start_timestamp + .saturating_duration_since(&meta.queued_timestamp); + CommandExecutionMetadata { wall_time: execution_time, execution_time, @@ -260,6 +266,7 @@ fn timing_from_re_metadata(meta: &TExecutedActionMetadata) -> CommandExecutionMe input_materialization_duration: fetch_input_time, hashing_duration: Duration::ZERO, hashed_artifacts_count: 0, + queue_duration: Some(queue_duration), } } diff --git a/app/buck2_execute_impl/src/executors/action_cache_upload_permission_checker.rs b/app/buck2_execute_impl/src/executors/action_cache_upload_permission_checker.rs index 863d96b41fb05..efbbc3d4ca63f 100644 --- a/app/buck2_execute_impl/src/executors/action_cache_upload_permission_checker.rs +++ b/app/buck2_execute_impl/src/executors/action_cache_upload_permission_checker.rs @@ -36,7 +36,7 @@ struct CacheValue { /// Check permission to upload to action cache and cache result. pub struct ActionCacheUploadPermissionChecker { re_client: ManagedRemoteExecutionClient, - /// Permission check does not depend on RE use case or platform, + /// Permission check does not depend on RE use case, /// but since we use these to upload, it is safer to cache the result by them. has_permission_to_upload_to_cache: DashMap>, } @@ -54,7 +54,7 @@ impl ActionCacheUploadPermissionChecker { re_use_case: RemoteExecutorUseCase, platform: &RePlatformFields, ) -> anyhow::Result> { - let (action, action_result) = empty_action_result()?; + let (action, action_result) = empty_action_result(platform)?; // This is CAS upload, if it fails, something is very broken. self.re_client diff --git a/app/buck2_execute_impl/src/executors/empty_action_result.rs b/app/buck2_execute_impl/src/executors/empty_action_result.rs index b296f73ffd2a7..4d67fec5e6895 100644 --- a/app/buck2_execute_impl/src/executors/empty_action_result.rs +++ b/app/buck2_execute_impl/src/executors/empty_action_result.rs @@ -8,6 +8,7 @@ */ use buck2_common::cas_digest::DigestAlgorithm; +use buck2_core::execution_types::executor_config::RePlatformFields; use buck2_execute::digest::CasDigestToReExt; use buck2_execute::digest_config::DigestConfig; use buck2_execute::execute::action_digest_and_blobs::ActionDigestAndBlobs; @@ -17,15 +18,12 @@ use remote_execution as RE; use remote_execution::TActionResult2; use remote_execution::TExecutedActionMetadata; -/// Create an empty action result for permission check. -pub(crate) fn empty_action_result() --> anyhow::Result<(&'static ActionDigestAndBlobs, &'static TActionResult2)> { - static EMPTY_ACTION_RESULT: OnceCell<(ActionDigestAndBlobs, TActionResult2)> = OnceCell::new(); - let (action, action_result) = EMPTY_ACTION_RESULT.get_or_try_init(new_empty_action_result)?; - Ok((action, action_result)) -} +use crate::executors::to_re_platform::RePlatformFieldsToRePlatform; -fn new_empty_action_result() -> anyhow::Result<(ActionDigestAndBlobs, TActionResult2)> { +/// Create an empty action result for permission check. +pub(crate) fn empty_action_result( + platform: &RePlatformFields, +) -> anyhow::Result<(ActionDigestAndBlobs, TActionResult2)> { static DIGEST_CONFIG: OnceCell = OnceCell::new(); let digest_config = *DIGEST_CONFIG .get_or_try_init(|| DigestConfig::leak_new(vec![DigestAlgorithm::Sha1], None))?; @@ -41,6 +39,7 @@ fn new_empty_action_result() -> anyhow::Result<(ActionDigestAndBlobs, TActionRes // Random string for xbgs. "EMPTY_ACTION_RESULT_fztiucvwawdmarhheqoz".to_owned(), ], + platform: Some(platform.to_re_platform()), ..Default::default() }); diff --git a/app/buck2_execute_impl/src/executors/local.rs b/app/buck2_execute_impl/src/executors/local.rs index 43526c066925b..d62238952aa97 100644 --- a/app/buck2_execute_impl/src/executors/local.rs +++ b/app/buck2_execute_impl/src/executors/local.rs @@ -450,6 +450,7 @@ impl LocalExecutor { input_materialization_duration, hashing_duration: Duration::ZERO, // We fill hashing info in later if available. hashed_artifacts_count: 0, + queue_duration: None, }; (timing, r) @@ -553,7 +554,9 @@ impl LocalExecutor { let (entry, hashing_info) = build_entry_from_disk( abspath, FileDigestConfig::build(digest_config.cas_digest_config()), + self.blocking_executor.as_ref(), ) + .await .with_context(|| format!("collecting output {:?}", path))?; total_hashing_time += hashing_info.hashing_duration; total_hashed_outputs += hashing_info.hashed_artifacts_count; diff --git a/app/buck2_execute_impl/src/materializers/deferred/mod.rs b/app/buck2_execute_impl/src/materializers/deferred/mod.rs index ed3b97519e2f7..1b0a1c9867e1a 100644 --- a/app/buck2_execute_impl/src/materializers/deferred/mod.rs +++ b/app/buck2_execute_impl/src/materializers/deferred/mod.rs @@ -64,6 +64,8 @@ use buck2_execute::output_size::OutputSize; use buck2_execute::re::manager::ReConnectionManager; use buck2_futures::cancellation::CancellationContext; use buck2_http::HttpClient; +use buck2_util::threads::check_stack_overflow; +use buck2_util::threads::thread_spawn; use buck2_wrapper_common::invocation_id::TraceId; use chrono::DateTime; use chrono::Duration; @@ -566,7 +568,20 @@ impl ArtifactMetadata { DirectoryEntry::Dir(DirectoryMetadata { fingerprint, .. }), DirectoryEntry::Dir(dir), ) => fingerprint == dir.fingerprint(), - (DirectoryEntry::Leaf(l1), DirectoryEntry::Leaf(l2)) => l1 == l2, + (DirectoryEntry::Leaf(l1), DirectoryEntry::Leaf(l2)) => { + // In Windows, the 'executable bit' absence can cause Buck2 to re-download identical artifacts. + // To avoid this, we exclude the executable bit from the comparison. + if cfg!(windows) { + match (l1, l2) { + ( + ActionDirectoryMember::File(meta1), + ActionDirectoryMember::File(meta2), + ) => return meta1.digest == meta2.digest, + _ => (), + } + } + l1 == l2 + } _ => false, } } @@ -1012,26 +1027,24 @@ impl DeferredMaterializerAccessor { let access_time_update_max_buffer_size = access_time_update_max_buffer_size()?; - let command_thread = std::thread::Builder::new() - .name("buck2-dm".to_owned()) - .spawn({ - move || { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - let cancellations = CancellationContext::never_cancelled(); - - rt.block_on(command_processor(cancellations).run( - command_receiver, - configs.ttl_refresh, - access_time_update_max_buffer_size, - configs.update_access_times, - )); - } - }) - .context("Cannot start materializer thread")?; + let command_thread = thread_spawn("buck2-dm", { + move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let cancellations = CancellationContext::never_cancelled(); + + rt.block_on(command_processor(cancellations).run( + command_receiver, + configs.ttl_refresh, + access_time_update_max_buffer_size, + configs.update_access_times, + )); + } + }) + .context("Cannot start materializer thread")?; Ok(Self { command_thread, @@ -1587,6 +1600,9 @@ impl DeferredMaterializerCommandProcessor { mut path: &ProjectRelativePath, event_dispatcher: EventDispatcher, ) -> Option { + // TODO(nga): rewrite without recursion. + check_stack_overflow().unwrap(); + // Get the data about the artifact, or return early if materializing/materialized let mut path_iter = path.iter(); let data = match self.tree.prefix_get_mut(&mut path_iter) { diff --git a/app/buck2_execute_impl/src/materializers/deferred/tests.rs b/app/buck2_execute_impl/src/materializers/deferred/tests.rs index 9a3e3043976d5..3ecf9a762db25 100644 --- a/app/buck2_execute_impl/src/materializers/deferred/tests.rs +++ b/app/buck2_execute_impl/src/materializers/deferred/tests.rs @@ -92,6 +92,7 @@ mod state_machine { use assert_matches::assert_matches; use buck2_execute::directory::Symlink; use buck2_execute::directory::INTERNER; + use buck2_util::threads::ignore_stack_overflow_checks_for_future; use parking_lot::Mutex; use tokio::time::sleep; use tokio::time::Duration as TokioDuration; @@ -292,52 +293,60 @@ mod state_machine { #[tokio::test] async fn test_declare_reuse() -> anyhow::Result<()> { - let (mut dm, _) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); + ignore_stack_overflow_checks_for_future(async { + let (mut dm, _) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); - let path = make_path("foo/bar"); - let value = ArtifactValue::file(digest_config.empty_file()); + let path = make_path("foo/bar"); + let value = ArtifactValue::file(digest_config.empty_file()); - dm.declare( - &path, - value.dupe(), - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, path.clone())]); - - let res = dm - .materialize_artifact(&path, EventDispatcher::null()) - .context("Expected a future")? - .await; - assert_eq!(dm.io.take_log(), &[(Op::Materialize, path.clone())]); - - dm.materialization_finished(path.clone(), Utc::now(), dm.version_tracker.current(), res); - assert_eq!(dm.io.take_log(), &[]); - - // When redeclaring the same artifact nothing happens. - dm.declare( - &path, - value.dupe(), - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[]); - - // When declaring the same artifact but under it, we clean it and it's a new artifact. - let path2 = make_path("foo/bar/baz"); - dm.declare( - &path2, - value.dupe(), - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, path2.clone())]); + dm.declare( + &path, + value.dupe(), + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[(Op::Clean, path.clone())]); + + let res = dm + .materialize_artifact(&path, EventDispatcher::null()) + .context("Expected a future")? + .await; + assert_eq!(dm.io.take_log(), &[(Op::Materialize, path.clone())]); + + dm.materialization_finished( + path.clone(), + Utc::now(), + dm.version_tracker.current(), + res, + ); + assert_eq!(dm.io.take_log(), &[]); + + // When redeclaring the same artifact nothing happens. + dm.declare( + &path, + value.dupe(), + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[]); + + // When declaring the same artifact but under it, we clean it and it's a new artifact. + let path2 = make_path("foo/bar/baz"); + dm.declare( + &path2, + value.dupe(), + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[(Op::Clean, path2.clone())]); - let _ignore = dm - .materialize_artifact(&path2, EventDispatcher::null()) - .context("Expected a future")? - .await; - assert_eq!(dm.io.take_log(), &[(Op::Materialize, path2.clone())]); + let _ignore = dm + .materialize_artifact(&path2, EventDispatcher::null()) + .context("Expected a future")? + .await; + assert_eq!(dm.io.take_log(), &[(Op::Materialize, path2.clone())]); - Ok(()) + Ok(()) + }) + .await } fn make_artifact_value_with_symlink_dep( @@ -364,133 +373,139 @@ mod state_machine { #[tokio::test] async fn test_materialize_symlink_and_target() -> anyhow::Result<()> { - // Construct a tree with a symlink and its target, materialize both at once - let symlink_path = make_path("foo/bar_symlink"); - let target_path = make_path("foo/bar_target"); - let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; - - let mut materialization_config = HashMap::new(); - // Materialize the symlink target slowly so that we actually hit the logic point where we - // await for symlink targets and the entry materialization - materialization_config.insert(target_path.clone(), TokioDuration::from_millis(100)); - - let (mut dm, _) = make_processor(materialization_config); - let digest_config = dm.io.digest_config(); - - // Declare symlink target - dm.declare( - &target_path, - ArtifactValue::file(digest_config.empty_file()), - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); - - // Declare symlink - let symlink_value = make_artifact_value_with_symlink_dep( - &target_path, - &target_from_symlink, - digest_config, - )?; - dm.declare( - &symlink_path, - symlink_value, - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, symlink_path.clone())]); - - dm.materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await - .map_err(|_| anyhow::anyhow!("error materializing"))?; - - let logs = dm.io.take_log(); - if cfg!(unix) { - assert_eq!( - logs, - &[ - (Op::Materialize, symlink_path.clone()), - (Op::Materialize, target_path.clone()) - ] + ignore_stack_overflow_checks_for_future(async { + // Construct a tree with a symlink and its target, materialize both at once + let symlink_path = make_path("foo/bar_symlink"); + let target_path = make_path("foo/bar_target"); + let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; + + let mut materialization_config = HashMap::new(); + // Materialize the symlink target slowly so that we actually hit the logic point where we + // await for symlink targets and the entry materialization + materialization_config.insert(target_path.clone(), TokioDuration::from_millis(100)); + + let (mut dm, _) = make_processor(materialization_config); + let digest_config = dm.io.digest_config(); + + // Declare symlink target + dm.declare( + &target_path, + ArtifactValue::file(digest_config.empty_file()), + Box::new(ArtifactMaterializationMethod::Test), ); - } else { - assert_eq!( - logs, - &[ - (Op::Materialize, target_path.clone()), - (Op::Materialize, symlink_path.clone()) - ] + assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); + + // Declare symlink + let symlink_value = make_artifact_value_with_symlink_dep( + &target_path, + &target_from_symlink, + digest_config, + )?; + dm.declare( + &symlink_path, + symlink_value, + Box::new(ArtifactMaterializationMethod::Test), ); - } - Ok(()) + assert_eq!(dm.io.take_log(), &[(Op::Clean, symlink_path.clone())]); + + dm.materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await + .map_err(|_| anyhow::anyhow!("error materializing"))?; + + let logs = dm.io.take_log(); + if cfg!(unix) { + assert_eq!( + logs, + &[ + (Op::Materialize, symlink_path.clone()), + (Op::Materialize, target_path.clone()) + ] + ); + } else { + assert_eq!( + logs, + &[ + (Op::Materialize, target_path.clone()), + (Op::Materialize, symlink_path.clone()) + ] + ); + } + Ok(()) + }) + .await } #[tokio::test] async fn test_materialize_symlink_first_then_target() -> anyhow::Result<()> { - // Materialize a symlink, then materialize the target. Test that we still - // materialize deps if the main artifact has already been materialized. - let symlink_path = make_path("foo/bar_symlink"); - let target_path = make_path("foo/bar_target"); - let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; - - let mut materialization_config = HashMap::new(); - // Materialize the symlink target slowly so that we actually hit the logic point where we - // await for symlink targets and the entry materialization - materialization_config.insert(target_path.clone(), TokioDuration::from_millis(100)); - - let (mut dm, _) = make_processor(materialization_config); - let digest_config = dm.io.digest_config(); - - // Declare symlink - let symlink_value = make_artifact_value_with_symlink_dep( - &target_path, - &target_from_symlink, - digest_config, - )?; - dm.declare( - &symlink_path, - symlink_value, - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, symlink_path.clone())]); - - // Materialize the symlink, at this point the target is not in the tree so it's ignored - let res = dm - .materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await; - - let logs = dm.io.take_log(); - assert_eq!(logs, &[(Op::Materialize, symlink_path.clone())]); - - // Mark the symlink as materialized - dm.materialization_finished( - symlink_path.clone(), - Utc::now(), - dm.version_tracker.current(), - res, - ); - assert_eq!(dm.io.take_log(), &[]); - - // Declare symlink target - dm.declare( - &target_path, - ArtifactValue::file(digest_config.empty_file()), - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); - - // Materialize the symlink again. - // This time, we don't re-materialize the symlink as that's already been done. - // But we still materialize the target as that has not been materialized yet. - dm.materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await - .map_err(|_| anyhow::anyhow!("error materializing"))?; - - let logs = dm.io.take_log(); - assert_eq!(logs, &[(Op::Materialize, target_path.clone())]); + ignore_stack_overflow_checks_for_future(async { + // Materialize a symlink, then materialize the target. Test that we still + // materialize deps if the main artifact has already been materialized. + let symlink_path = make_path("foo/bar_symlink"); + let target_path = make_path("foo/bar_target"); + let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; + + let mut materialization_config = HashMap::new(); + // Materialize the symlink target slowly so that we actually hit the logic point where we + // await for symlink targets and the entry materialization + materialization_config.insert(target_path.clone(), TokioDuration::from_millis(100)); + + let (mut dm, _) = make_processor(materialization_config); + let digest_config = dm.io.digest_config(); + + // Declare symlink + let symlink_value = make_artifact_value_with_symlink_dep( + &target_path, + &target_from_symlink, + digest_config, + )?; + dm.declare( + &symlink_path, + symlink_value, + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[(Op::Clean, symlink_path.clone())]); + + // Materialize the symlink, at this point the target is not in the tree so it's ignored + let res = dm + .materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await; + + let logs = dm.io.take_log(); + assert_eq!(logs, &[(Op::Materialize, symlink_path.clone())]); + + // Mark the symlink as materialized + dm.materialization_finished( + symlink_path.clone(), + Utc::now(), + dm.version_tracker.current(), + res, + ); + assert_eq!(dm.io.take_log(), &[]); - Ok(()) + // Declare symlink target + dm.declare( + &target_path, + ArtifactValue::file(digest_config.empty_file()), + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); + + // Materialize the symlink again. + // This time, we don't re-materialize the symlink as that's already been done. + // But we still materialize the target as that has not been materialized yet. + dm.materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await + .map_err(|_| anyhow::anyhow!("error materializing"))?; + + let logs = dm.io.take_log(); + assert_eq!(logs, &[(Op::Materialize, target_path.clone())]); + + Ok(()) + }) + .await } #[tokio::test] @@ -516,315 +531,330 @@ mod state_machine { #[tokio::test] async fn test_subscription_notifications() { - let (mut dm, mut channel) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); - let value = ArtifactValue::file(digest_config.empty_file()); - - let mut handle = { - let (sender, recv) = oneshot::channel(); - MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); - recv.await.unwrap() - }; - - let foo_bar = make_path("foo/bar"); - let foo_bar_baz = make_path("foo/bar/baz"); - let bar = make_path("bar"); - let qux = make_path("qux"); - - dm.declare_existing(&foo_bar, value.dupe()); - - handle.subscribe_to_paths(vec![foo_bar_baz.clone(), bar.clone()]); - while let Ok(cmd) = channel.high_priority.try_recv() { - dm.process_one_command(cmd); - } + ignore_stack_overflow_checks_for_future(async { + let (mut dm, mut channel) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); + let value = ArtifactValue::file(digest_config.empty_file()); + + let mut handle = { + let (sender, recv) = oneshot::channel(); + MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); + recv.await.unwrap() + }; + + let foo_bar = make_path("foo/bar"); + let foo_bar_baz = make_path("foo/bar/baz"); + let bar = make_path("bar"); + let qux = make_path("qux"); + + dm.declare_existing(&foo_bar, value.dupe()); + + handle.subscribe_to_paths(vec![foo_bar_baz.clone(), bar.clone()]); + while let Ok(cmd) = channel.high_priority.try_recv() { + dm.process_one_command(cmd); + } - dm.declare_existing(&bar, value.dupe()); - dm.declare_existing(&foo_bar_baz, value.dupe()); - dm.declare_existing(&qux, value.dupe()); + dm.declare_existing(&bar, value.dupe()); + dm.declare_existing(&foo_bar_baz, value.dupe()); + dm.declare_existing(&qux, value.dupe()); - let mut paths = Vec::new(); - while let Ok(path) = handle.receiver().try_recv() { - paths.push(path); - } + let mut paths = Vec::new(); + while let Ok(path) = handle.receiver().try_recv() { + paths.push(path); + } - assert_eq!(paths, vec![foo_bar_baz.clone(), bar, foo_bar_baz]); + assert_eq!(paths, vec![foo_bar_baz.clone(), bar, foo_bar_baz]); + }) + .await } #[tokio::test] async fn test_subscription_subscribe_also_materializes() -> anyhow::Result<()> { - let (mut dm, mut channel) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); - let value = ArtifactValue::file(digest_config.empty_file()); - - let mut handle = { - let (sender, recv) = oneshot::channel(); - MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); - recv.await.unwrap() - }; - - let foo_bar = make_path("foo/bar"); - - dm.declare( - &foo_bar, - value.dupe(), - Box::new(ArtifactMaterializationMethod::Test), - ); + ignore_stack_overflow_checks_for_future(async { + let (mut dm, mut channel) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); + let value = ArtifactValue::file(digest_config.empty_file()); + + let mut handle = { + let (sender, recv) = oneshot::channel(); + MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); + recv.await.unwrap() + }; + + let foo_bar = make_path("foo/bar"); + + dm.declare( + &foo_bar, + value.dupe(), + Box::new(ArtifactMaterializationMethod::Test), + ); - handle.subscribe_to_paths(vec![foo_bar.clone()]); - while let Ok(cmd) = channel.high_priority.try_recv() { - dm.process_one_command(cmd); - } + handle.subscribe_to_paths(vec![foo_bar.clone()]); + while let Ok(cmd) = channel.high_priority.try_recv() { + dm.process_one_command(cmd); + } - // We need to yield to let the materialization task run. If we had a handle to it, we'd - // just await it, but the subscription isn't retaining those handles. - let mut log = Vec::new(); - while log.len() < 2 { - log.extend(dm.io.take_log()); - tokio::task::yield_now().await; - } + // We need to yield to let the materialization task run. If we had a handle to it, we'd + // just await it, but the subscription isn't retaining those handles. + let mut log = Vec::new(); + while log.len() < 2 { + log.extend(dm.io.take_log()); + tokio::task::yield_now().await; + } - assert_eq!( - &log, - &[ - (Op::Clean, foo_bar.clone()), - (Op::Materialize, foo_bar.clone()) - ] - ); + assert_eq!( + &log, + &[ + (Op::Clean, foo_bar.clone()), + (Op::Materialize, foo_bar.clone()) + ] + ); - // Drain low priority commands. This should include our materialization finished message, - // at which point we'll notify the subscription handle. - while let Ok(cmd) = channel.low_priority.try_recv() { - dm.process_one_low_priority_command(cmd); - } + // Drain low priority commands. This should include our materialization finished message, + // at which point we'll notify the subscription handle. + while let Ok(cmd) = channel.low_priority.try_recv() { + dm.process_one_low_priority_command(cmd); + } - let mut paths = Vec::new(); - while let Ok(path) = handle.receiver().try_recv() { - paths.push(path); - } + let mut paths = Vec::new(); + while let Ok(path) = handle.receiver().try_recv() { + paths.push(path); + } - assert_eq!(paths, vec![foo_bar]); + assert_eq!(paths, vec![foo_bar]); - Ok(()) + Ok(()) + }) + .await } #[tokio::test] async fn test_subscription_unsubscribe() { - let (mut dm, mut channel) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); - let value1 = ArtifactValue::file(digest_config.empty_file()); - let value2 = ArtifactValue::dir(digest_config.empty_directory()); - - let mut handle = { - let (sender, recv) = oneshot::channel(); - MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); - recv.await.unwrap() - }; - - let path = make_path("foo/bar"); - - handle.subscribe_to_paths(vec![path.clone()]); - while let Ok(cmd) = channel.high_priority.try_recv() { - dm.process_one_command(cmd); - } + ignore_stack_overflow_checks_for_future(async { + let (mut dm, mut channel) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); + let value1 = ArtifactValue::file(digest_config.empty_file()); + let value2 = ArtifactValue::dir(digest_config.empty_directory()); + + let mut handle = { + let (sender, recv) = oneshot::channel(); + MaterializerSubscriptionOperation::Create { sender }.execute(&mut dm); + recv.await.unwrap() + }; + + let path = make_path("foo/bar"); + + handle.subscribe_to_paths(vec![path.clone()]); + while let Ok(cmd) = channel.high_priority.try_recv() { + dm.process_one_command(cmd); + } - dm.declare_existing(&path, value1.dupe()); + dm.declare_existing(&path, value1.dupe()); - handle.unsubscribe_from_paths(vec![path.clone()]); - while let Ok(cmd) = channel.high_priority.try_recv() { - dm.process_one_command(cmd); - } + handle.unsubscribe_from_paths(vec![path.clone()]); + while let Ok(cmd) = channel.high_priority.try_recv() { + dm.process_one_command(cmd); + } - dm.declare_existing(&path, value2.dupe()); + dm.declare_existing(&path, value2.dupe()); - let mut paths = Vec::new(); - while let Ok(path) = handle.receiver().try_recv() { - paths.push(path); - } + let mut paths = Vec::new(); + while let Ok(path) = handle.receiver().try_recv() { + paths.push(path); + } - // Expect only one notification - assert_eq!(paths, vec![path]); + // Expect only one notification + assert_eq!(paths, vec![path]); + }) + .await } #[tokio::test] async fn test_invalidate_error() -> anyhow::Result<()> { - let (mut dm, _) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); + ignore_stack_overflow_checks_for_future(async{ + let (mut dm, _) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); - let path = make_path("test/invalidate/failure"); - let value1 = ArtifactValue::file(digest_config.empty_file()); - let value2 = ArtifactValue::dir(digest_config.empty_directory()); + let path = make_path("test/invalidate/failure"); + let value1 = ArtifactValue::file(digest_config.empty_file()); + let value2 = ArtifactValue::dir(digest_config.empty_directory()); - // Start from having something. - dm.declare_existing(&path, value1); + // Start from having something. + dm.declare_existing(&path, value1); - // This will collect the existing future and invalidate, and then fail in doing so. - dm.declare(&path, value2, Box::new(ArtifactMaterializationMethod::Test)); + // This will collect the existing future and invalidate, and then fail in doing so. + dm.declare(&path, value2, Box::new(ArtifactMaterializationMethod::Test)); - // Now we check that materialization fails. This needs to wait on the previous clean. - let res = dm - .materialize_artifact(&path, EventDispatcher::null()) - .context("Expected a future")? - .await; + // Now we check that materialization fails. This needs to wait on the previous clean. + let res = dm + .materialize_artifact(&path, EventDispatcher::null()) + .context("Expected a future")? + .await; - assert_matches!( + assert_matches!( res, Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") ); - // We do not actually get to materializing or cleaning. - assert_eq!(dm.io.take_log(), &[]); + // We do not actually get to materializing or cleaning. + assert_eq!(dm.io.take_log(), &[]); - Ok(()) + Ok(()) + }).await } #[tokio::test] async fn test_materialize_dep_error() -> anyhow::Result<()> { - // Construct a tree with a symlink and its target, materialize both at once - let symlink_path = make_path("foo/bar_symlink"); - let target_path = make_path("foo/bar_target"); - let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; - - let (mut dm, mut channel) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); - - let target_value = ArtifactValue::file(digest_config.empty_file()); - let symlink_value = make_artifact_value_with_symlink_dep( - &target_path, - &target_from_symlink, - digest_config, - )?; - // Declare and materialize symlink and target - dm.declare( - &target_path, - target_value.clone(), - Box::new(ArtifactMaterializationMethod::Test), - ); - dm.declare( - &symlink_path, - symlink_value.clone(), - Box::new(ArtifactMaterializationMethod::Test), - ); - dm.materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await - .map_err(|err| anyhow::anyhow!("error materializing {:?}", err))?; - assert_eq!( - dm.io.take_log(), - &[ - (Op::Clean, target_path.clone()), - (Op::Clean, symlink_path.clone()), - (Op::Materialize, target_path.clone()), - (Op::Materialize, symlink_path.clone()), - ] - ); + ignore_stack_overflow_checks_for_future(async { + // Construct a tree with a symlink and its target, materialize both at once + let symlink_path = make_path("foo/bar_symlink"); + let target_path = make_path("foo/bar_target"); + let target_from_symlink = RelativePathBuf::from_path(Path::new("bar_target"))?; + + let (mut dm, mut channel) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); + + let target_value = ArtifactValue::file(digest_config.empty_file()); + let symlink_value = make_artifact_value_with_symlink_dep( + &target_path, + &target_from_symlink, + digest_config, + )?; + // Declare and materialize symlink and target + dm.declare( + &target_path, + target_value.clone(), + Box::new(ArtifactMaterializationMethod::Test), + ); + dm.declare( + &symlink_path, + symlink_value.clone(), + Box::new(ArtifactMaterializationMethod::Test), + ); + dm.materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await + .map_err(|err| anyhow::anyhow!("error materializing {:?}", err))?; + assert_eq!( + dm.io.take_log(), + &[ + (Op::Clean, target_path.clone()), + (Op::Clean, symlink_path.clone()), + (Op::Materialize, target_path.clone()), + (Op::Materialize, symlink_path.clone()), + ] + ); - // Process materialization_finished, change symlink stage to materialized - while let Ok(cmd) = channel.low_priority.try_recv() { - dm.process_one_low_priority_command(cmd); - } + // Process materialization_finished, change symlink stage to materialized + while let Ok(cmd) = channel.low_priority.try_recv() { + dm.process_one_low_priority_command(cmd); + } - // Change symlink target value and re-declare - let content = b"not empty"; - let meta = FileMetadata { - digest: TrackedFileDigest::from_content(content, digest_config.cas_digest_config()), - is_executable: false, - }; - let target_value = ArtifactValue::file(meta); - dm.declare( - &target_path, - target_value, - Box::new(ArtifactMaterializationMethod::Test), - ); - assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); - - // Request to materialize symlink, fail to materialize target - dm.io.set_fail_on(vec![target_path.clone()]); - let res = dm - .materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await; - assert_matches!( + // Change symlink target value and re-declare + let content = b"not empty"; + let meta = FileMetadata { + digest: TrackedFileDigest::from_content(content, digest_config.cas_digest_config()), + is_executable: false, + }; + let target_value = ArtifactValue::file(meta); + dm.declare( + &target_path, + target_value, + Box::new(ArtifactMaterializationMethod::Test), + ); + assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); + + // Request to materialize symlink, fail to materialize target + dm.io.set_fail_on(vec![target_path.clone()]); + let res = dm + .materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await; + assert_matches!( res, Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") ); - assert_eq!( - dm.io.take_log(), - &[(Op::MaterializeError, target_path.clone())] - ); - // Process materialization_finished, _only_ target is cleaned, not symlink - while let Ok(cmd) = channel.low_priority.try_recv() { - dm.process_one_low_priority_command(cmd); - } - assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); - - // Request symlink again, target is materialized and symlink materialization succeeds - dm.io.set_fail_on(vec![]); - dm.materialize_artifact(&symlink_path, EventDispatcher::null()) - .context("Expected a future")? - .await - .map_err(|err| anyhow::anyhow!("error materializing 2 {:?}", err))?; - assert_eq!(dm.io.take_log(), &[(Op::Materialize, target_path.clone()),]); - Ok(()) + assert_eq!( + dm.io.take_log(), + &[(Op::MaterializeError, target_path.clone())] + ); + // Process materialization_finished, _only_ target is cleaned, not symlink + while let Ok(cmd) = channel.low_priority.try_recv() { + dm.process_one_low_priority_command(cmd); + } + assert_eq!(dm.io.take_log(), &[(Op::Clean, target_path.clone())]); + + // Request symlink again, target is materialized and symlink materialization succeeds + dm.io.set_fail_on(vec![]); + dm.materialize_artifact(&symlink_path, EventDispatcher::null()) + .context("Expected a future")? + .await + .map_err(|err| anyhow::anyhow!("error materializing 2 {:?}", err))?; + assert_eq!(dm.io.take_log(), &[(Op::Materialize, target_path.clone()), ]); + Ok(()) + }).await } #[tokio::test] async fn test_retry() -> anyhow::Result<()> { - let (mut dm, mut channel) = make_processor(Default::default()); - let digest_config = dm.io.digest_config(); + ignore_stack_overflow_checks_for_future(async { + let (mut dm, mut channel) = make_processor(Default::default()); + let digest_config = dm.io.digest_config(); - let path = make_path("test"); - let value1 = ArtifactValue::file(digest_config.empty_file()); + let path = make_path("test"); + let value1 = ArtifactValue::file(digest_config.empty_file()); - // Declare a value. - dm.declare(&path, value1, Box::new(ArtifactMaterializationMethod::Test)); + // Declare a value. + dm.declare(&path, value1, Box::new(ArtifactMaterializationMethod::Test)); - // Make materializations fail - dm.io.set_fail(true); + // Make materializations fail + dm.io.set_fail(true); - // Materializing it fails. - let res = dm - .materialize_artifact(&path, EventDispatcher::null()) - .context("Expected a future")? - .await; + // Materializing it fails. + let res = dm + .materialize_artifact(&path, EventDispatcher::null()) + .context("Expected a future")? + .await; - assert_matches!( - res, - Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") - ); + assert_matches!( + res, + Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") + ); - // Unset fail, but we haven't processed materialization_finished yet so this does nothing. - dm.io.set_fail(false); + // Unset fail, but we haven't processed materialization_finished yet so this does nothing. + dm.io.set_fail(false); - // Rejoining the existing future fails. - let res = dm - .materialize_artifact(&path, EventDispatcher::null()) - .context("Expected a future")? - .await; + // Rejoining the existing future fails. + let res = dm + .materialize_artifact(&path, EventDispatcher::null()) + .context("Expected a future")? + .await; - assert_matches!( - res, - Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") - ); + assert_matches!( + res, + Err(SharedMaterializingError::Error(e)) if format!("{:#}", e).contains("Injected error") + ); - // Now process cleanup_finished_vacant and materialization_finished. - let mut processed = 0; + // Now process cleanup_finished_vacant and materialization_finished. + let mut processed = 0; - while let Ok(cmd) = channel.low_priority.try_recv() { - eprintln!("got cmd = {:?}", cmd); - dm.process_one_low_priority_command(cmd); - processed += 1; - } + while let Ok(cmd) = channel.low_priority.try_recv() { + eprintln!("got cmd = {:?}", cmd); + dm.process_one_low_priority_command(cmd); + processed += 1; + } - assert_eq!(processed, 2); + assert_eq!(processed, 2); - // Materializing works now: - let res = dm - .materialize_artifact(&path, EventDispatcher::null()) - .context("Expected a future")? - .await; + // Materializing works now: + let res = dm + .materialize_artifact(&path, EventDispatcher::null()) + .context("Expected a future")? + .await; - assert_matches!(res, Ok(())); + assert_matches!(res, Ok(())); - Ok(()) + Ok(()) + }).await } } diff --git a/app/buck2_forkserver/BUCK b/app/buck2_forkserver/BUCK index a8979ed1d936d..af80b9714a469 100644 --- a/app/buck2_forkserver/BUCK +++ b/app/buck2_forkserver/BUCK @@ -65,6 +65,7 @@ rust_library( "fbsource//third-party/rust:libc", "fbsource//third-party/rust:pin-project", "fbsource//third-party/rust:take_mut", + "fbsource//third-party/rust:thiserror", "fbsource//third-party/rust:tokio", "fbsource//third-party/rust:tokio-util", "fbsource//third-party/rust:tonic", diff --git a/app/buck2_forkserver/Cargo.toml b/app/buck2_forkserver/Cargo.toml index a91a070e08626..8529094f9ef51 100644 --- a/app/buck2_forkserver/Cargo.toml +++ b/app/buck2_forkserver/Cargo.toml @@ -15,6 +15,7 @@ futures = { workspace = true } libc = { workspace = true } pin-project = { workspace = true } take_mut = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } tonic = { workspace = true } diff --git a/app/buck2_forkserver/src/convert.rs b/app/buck2_forkserver/src/convert.rs index 393304d2fa495..f78a333b27b3a 100644 --- a/app/buck2_forkserver/src/convert.rs +++ b/app/buck2_forkserver/src/convert.rs @@ -15,7 +15,8 @@ use futures::stream::StreamExt; use crate::run::CommandEvent; use crate::run::GatherOutputStatus; -pub fn encode_event_stream( +#[allow(dead_code)] +pub(crate) fn encode_event_stream( s: S, ) -> impl Stream> where @@ -61,7 +62,7 @@ where s.map(|r| r.map(convert_event).map_err(convert_err)) } -pub fn decode_event_stream(s: S) -> impl Stream> +pub(crate) fn decode_event_stream(s: S) -> impl Stream> where S: Stream>, { diff --git a/app/buck2_forkserver/src/lib.rs b/app/buck2_forkserver/src/lib.rs index f1f5209ee75a1..26d4ba45a5e74 100644 --- a/app/buck2_forkserver/src/lib.rs +++ b/app/buck2_forkserver/src/lib.rs @@ -8,10 +8,10 @@ */ #![feature(error_generic_member_access)] -#![feature(async_fn_in_trait)] #![feature(offset_of)] +#![cfg_attr(windows, feature(windows_process_extensions_main_thread_handle))] pub mod client; -pub mod convert; +pub(crate) mod convert; pub mod run; #[cfg(unix)] diff --git a/app/buck2_forkserver/src/run/interruptible_async_read.rs b/app/buck2_forkserver/src/run/interruptible_async_read.rs index d1bcb6f5cf9bc..18d5298c2333c 100644 --- a/app/buck2_forkserver/src/run/interruptible_async_read.rs +++ b/app/buck2_forkserver/src/run/interruptible_async_read.rs @@ -28,12 +28,12 @@ use tokio_util::codec::FramedRead; /// This trait represents the ability to transition a Reader (R) to a Drainer (Self). Both are /// [AsyncRead], but we expect the Drainer (which implements this trait) to not wait longer for /// more data to be produced. -pub trait DrainerFromReader { +pub(crate) trait DrainerFromReader { fn from_reader(reader: R) -> Self; } /// This trait represents a AsyncRead that can be told to interrupt (and transition to draining). -pub trait InterruptNotifiable { +pub(crate) trait InterruptNotifiable { fn notify_interrupt(self: Pin<&mut Self>); } @@ -42,7 +42,7 @@ pub trait InterruptNotifiable { /// proceed to "drain" the reader, which means reading the data that is there but not waiting for /// any further data to get written. #[pin_project] -pub struct InterruptibleAsyncRead { +pub(crate) struct InterruptibleAsyncRead { state: InterruptibleAsyncReadState, } @@ -95,7 +95,7 @@ mod unix_non_blocking_drainer { /// hasn't completed `select()` on the pipe we want to drain, we'll still get to execute /// `read()` (and potentially get WouldBlock if there is nothing to read and the pipe isn't /// ready). - pub struct UnixNonBlockingDrainer { + pub(crate) struct UnixNonBlockingDrainer { fd: RawFd, // Kept so this is dropped and closed properly. _owner: R, @@ -140,10 +140,10 @@ mod unix_non_blocking_drainer { } #[cfg(unix)] -pub use unix_non_blocking_drainer::*; +pub(crate) use unix_non_blocking_drainer::*; #[pin_project] -pub struct TimeoutDrainer { +pub(crate) struct TimeoutDrainer { state: TimeoutDrainerState, // To have a generic parameter like UnixNonBlockingDrainer does. _phantom: PhantomData, diff --git a/app/buck2_forkserver/src/run/mod.rs b/app/buck2_forkserver/src/run/mod.rs index 5c9cd1cdc48bd..cc06fa7611a2e 100644 --- a/app/buck2_forkserver/src/run/mod.rs +++ b/app/buck2_forkserver/src/run/mod.rs @@ -16,7 +16,6 @@ use std::path::Path; use std::pin::Pin; use std::process::Command; use std::process::ExitStatus; -use std::process::Stdio; use std::task::Context; use std::task::Poll; use std::time::Duration; @@ -40,7 +39,9 @@ use self::interruptible_async_read::InterruptibleAsyncRead; use self::status_decoder::DecodedStatus; use self::status_decoder::DefaultStatusDecoder; use self::status_decoder::StatusDecoder; +use crate::run::process_group::ProcessCommand; use crate::run::process_group::ProcessGroup; +use crate::run::process_group::SpawnError; #[derive(Debug)] pub enum GatherOutputStatus { @@ -70,7 +71,7 @@ impl From for GatherOutputStatus { } #[derive(Debug)] -pub enum CommandEvent { +pub(crate) enum CommandEvent { Stdout(Bytes), Stderr(Bytes), Exit(GatherOutputStatus), @@ -171,7 +172,7 @@ pub async fn timeout_into_cancellation( } } -pub fn stream_command_events( +pub(crate) fn stream_command_events( process_group: anyhow::Result, cancellation: T, decoder: impl StatusDecoder, @@ -193,14 +194,10 @@ where let stdio = if stream_stdio { let stdout = process_group - .child() - .stdout - .take() + .take_stdout() .context("Child stdout is not piped")?; let stderr = process_group - .child() - .stderr - .take() + .take_stderr() .context("Child stderr is not piped")?; #[cfg(unix)] @@ -232,7 +229,7 @@ where // NOTE: This wrapping here is so that we release the borrow of `child` that stems from // `wait()` by the time we call kill_process a few lines down. let execute = async { - let status = process_group.child().wait(); + let status = process_group.wait(); futures::pin_mut!(status); futures::pin_mut!(cancellation); @@ -258,7 +255,6 @@ where // We just killed the child, so this should finish immediately. We should still call // this to release any process. process_group - .child() .wait() .await .context("Failed to await child after kill")?; @@ -302,7 +298,7 @@ pub async fn gather_output( where T: Future> + Send, { - let cmd = prepare_command(cmd); + let cmd = ProcessCommand::new(cmd); let process_details = spawn_retry_txt_busy(cmd, || tokio::time::sleep(Duration::from_millis(50))).await; @@ -319,19 +315,19 @@ where /// Dependency injection for kill. We use this in testing. #[async_trait] -pub trait KillProcess { +pub(crate) trait KillProcess { async fn kill(self, process: &mut ProcessGroup) -> anyhow::Result<()>; } #[derive(Default)] -pub struct DefaultKillProcess { +pub(crate) struct DefaultKillProcess { pub graceful_shutdown_timeout_s: Option, } #[async_trait] impl KillProcess for DefaultKillProcess { async fn kill(self, process_group: &mut ProcessGroup) -> anyhow::Result<()> { - let pid = match process_group.child().id() { + let pid = match process_group.id() { Some(pid) => pid, None => { // Child just exited, so in this case we don't want to kill anything. @@ -339,15 +335,7 @@ impl KillProcess for DefaultKillProcess { } }; tracing::info!("Killing process {}", pid); - - if cfg!(any(windows, unix)) { - return process_group.kill(self.graceful_shutdown_timeout_s).await; - } - - process_group - .child() - .start_kill() - .map_err(anyhow::Error::from) + process_group.kill(self.graceful_shutdown_timeout_s).await } } @@ -371,29 +359,6 @@ pub fn maybe_absolutize_exe<'a>( Ok(exe.into()) } -pub fn prepare_command(mut cmd: Command) -> tokio::process::Command { - #[cfg(unix)] - { - use std::os::unix::process::CommandExt; - cmd.process_group(0); - } - - #[cfg(windows)] - { - // On windows we create suspended process to assign it to a job (group) and then resume. - // This is necessary because the process might finish before we add it to a job - use std::os::windows::process::CommandExt; - cmd.creation_flags( - winapi::um::winbase::CREATE_NO_WINDOW | winapi::um::winbase::CREATE_SUSPENDED, - ); - } - - cmd.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - cmd.into() -} - /// fork-exec is a bit tricky in a busy process. We often have files open to writing just prior to /// executing them (as we download from RE), and many processes being spawned concurrently. We do /// close the fds properly before the exec, but what can happn is: @@ -410,7 +375,7 @@ pub fn prepare_command(mut cmd: Command) -> tokio::process::Command { /// The more correct solution for this here would be to start a fork server in a separate process /// when we start. However, until we get there, this should do the trick. async fn spawn_retry_txt_busy( - mut cmd: tokio::process::Command, + mut cmd: ProcessCommand, mut delay: F, ) -> anyhow::Result where @@ -422,11 +387,14 @@ where loop { let res = cmd.spawn(); - let res_errno = res.as_ref().map_err(|e| e.raw_os_error()); + let res_errno = res.as_ref().map_err(|e| match e { + SpawnError::IoError(e) => e.raw_os_error(), + SpawnError::GenericError(_) => None, + }); let is_txt_busy = matches!(res_errno, Err(Some(libc::ETXTBSY))); if attempts == 0 || !is_txt_busy { - return res.map_err(anyhow::Error::from).and_then(ProcessGroup::new); + return res.map_err(anyhow::Error::from); } delay().await; @@ -444,7 +412,6 @@ mod tests { use std::time::Instant; use assert_matches::assert_matches; - use buck2_util::process::async_background_command; use buck2_util::process::background_command; use dupe::Dupe; @@ -546,7 +513,8 @@ mod tests { file.write_all(b"#!/usr/bin/env bash\ntrue\n").await?; - let cmd = async_background_command(&bin); + let cmd = background_command(&bin); + let cmd = ProcessCommand::new(cmd); let mut process_group = spawn_retry_txt_busy(cmd, { let mut file = Some(file); move || { @@ -556,7 +524,7 @@ mod tests { }) .await?; - let status = process_group.child().wait().await?; + let status = process_group.wait().await?; assert_eq!(status.code(), Some(0)); Ok(()) @@ -567,7 +535,8 @@ mod tests { let tempdir = tempfile::tempdir()?; let bin = tempdir.path().join("bin"); // Does not actually exist - let cmd = async_background_command(&bin); + let cmd = background_command(&bin); + let cmd = ProcessCommand::new(cmd); let res = spawn_retry_txt_busy(cmd, || async { panic!("Should not be called!") }).await; assert!(res.is_err()); @@ -630,10 +599,8 @@ mod tests { }; cmd.args(["-c", "exit 0"]); - let process = prepare_command(cmd) - .spawn() - .map_err(anyhow::Error::from) - .and_then(ProcessGroup::new); + let mut cmd = ProcessCommand::new(cmd); + let process = cmd.spawn().map_err(anyhow::Error::from); let mut events = stream_command_events( process, futures::future::pending(), @@ -709,11 +676,8 @@ mod tests { }; cmd.args(["-c", "sleep 10000"]); - let mut cmd = prepare_command(cmd); - let process = cmd - .spawn() - .map_err(anyhow::Error::from) - .and_then(ProcessGroup::new); + let mut cmd = ProcessCommand::new(cmd); + let process = cmd.spawn().map_err(anyhow::Error::from); let stream = stream_command_events( process, @@ -743,15 +707,12 @@ mod tests { let mut cmd = background_command("sh"); cmd.args(["-c", "echo hello"]); - let mut cmd = prepare_command(cmd); let tempdir = tempfile::tempdir()?; let stdout = tempdir.path().join("stdout"); + let mut cmd = ProcessCommand::new(cmd); cmd.stdout(std::fs::File::create(stdout.clone())?); - let process_group = cmd - .spawn() - .map_err(anyhow::Error::from) - .and_then(ProcessGroup::new); + let process_group = cmd.spawn().map_err(anyhow::Error::from); let mut events = stream_command_events( process_group, futures::future::pending(), diff --git a/app/buck2_forkserver/src/run/process_group.rs b/app/buck2_forkserver/src/run/process_group.rs index 1855f2ef49f2a..da49463294702 100644 --- a/app/buck2_forkserver/src/run/process_group.rs +++ b/app/buck2_forkserver/src/run/process_group.rs @@ -7,29 +7,126 @@ * of this source tree. */ -use tokio::process::Child; +use std::process::Command as StdCommand; +use std::process::ExitStatus; +use std::process::Stdio; + +use thiserror::Error; +use tokio::io; +use tokio::process::ChildStderr; +use tokio::process::ChildStdout; #[cfg(unix)] use crate::unix::process_group as imp; #[cfg(windows)] use crate::win::process_group as imp; -pub struct ProcessGroup { - inner: imp::ProcessGroupImpl, +#[derive(Error, Debug)] +pub(crate) enum SpawnError { + #[error("Failed to spawn a process")] + IoError(#[from] io::Error), + #[error("Failed to create a process group")] + GenericError(#[from] anyhow::Error), } -impl ProcessGroup { - pub fn new(child: Child) -> anyhow::Result { +pub(crate) struct ProcessCommand { + inner: imp::ProcessCommandImpl, +} + +impl ProcessCommand { + pub(crate) fn new(mut cmd: StdCommand) -> Self { + cmd.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + Self { + inner: imp::ProcessCommandImpl::new(cmd), + } + } + + pub(crate) fn spawn(&mut self) -> anyhow::Result { + let child = self.inner.spawn()?; Ok(ProcessGroup { inner: imp::ProcessGroupImpl::new(child)?, }) } - pub fn child(&mut self) -> &mut Child { - self.inner.child() + #[allow(dead_code)] + pub(crate) fn stdout>(&mut self, cfg: T) -> &mut ProcessCommand { + self.inner.stdout(cfg.into()); + self } - pub async fn kill(&self, graceful_shutdown_timeout_s: Option) -> anyhow::Result<()> { + #[allow(dead_code)] + pub(crate) fn stderr>(&mut self, cfg: T) -> &mut ProcessCommand { + self.inner.stderr(cfg.into()); + self + } +} + +pub(crate) struct ProcessGroup { + inner: imp::ProcessGroupImpl, +} + +impl ProcessGroup { + pub(crate) fn take_stdout(&mut self) -> Option { + self.inner.take_stdout() + } + + pub(crate) fn take_stderr(&mut self) -> Option { + self.inner.take_stderr() + } + + pub(crate) async fn wait(&mut self) -> io::Result { + self.inner.wait().await + } + + pub(crate) fn id(&self) -> Option { + self.inner.id() + } + + pub(crate) async fn kill( + &self, + graceful_shutdown_timeout_s: Option, + ) -> anyhow::Result<()> { self.inner.kill(graceful_shutdown_timeout_s).await } } + +#[cfg(test)] +mod tests { + use buck2_util::process::background_command; + + use crate::run::process_group::ProcessCommand; + + // The test check basic functionality of process implementation as it differs on Unix and Windows + #[tokio::test] + async fn test_process_impl() -> anyhow::Result<()> { + let mut cmd; + + if cfg!(windows) { + cmd = background_command("cmd"); + cmd.arg("/c"); + } else { + cmd = background_command("sh"); + cmd.arg("-c"); + } + cmd.arg("exit 2"); + + let mut cmd = ProcessCommand::new(cmd); + let mut child = cmd.spawn().unwrap(); + + let id = child.id().expect("missing id"); + assert!(id > 0); + + let status = child.wait().await?; + assert_eq!(status.code(), Some(2)); + + // test that the `.wait()` method is fused like tokio + let status = child.wait().await?; + assert_eq!(status.code(), Some(2)); + + // Can't get id after process has exited + assert_eq!(child.id(), None); + Ok(()) + } +} diff --git a/app/buck2_forkserver/src/run/status_decoder.rs b/app/buck2_forkserver/src/run/status_decoder.rs index 59791da60d6a3..1263561078689 100644 --- a/app/buck2_forkserver/src/run/status_decoder.rs +++ b/app/buck2_forkserver/src/run/status_decoder.rs @@ -14,7 +14,7 @@ use async_trait::async_trait; use buck2_core::fs::paths::abs_norm_path::AbsNormPathBuf; use buck2_miniperf_proto::MiniperfOutput; -pub enum DecodedStatus { +pub(crate) enum DecodedStatus { /// An actual status. Status { exit_code: i32, @@ -26,7 +26,7 @@ pub enum DecodedStatus { } #[async_trait] -pub trait StatusDecoder { +pub(crate) trait StatusDecoder { /// Status decoders receive the exit status of the command we ran, but they might also obtain /// information out of band to obtain a different exit status. async fn decode_status(self, status: ExitStatus) -> anyhow::Result; @@ -35,7 +35,7 @@ pub trait StatusDecoder { async fn cancel(self) -> anyhow::Result<()>; } -pub struct DefaultStatusDecoder; +pub(crate) struct DefaultStatusDecoder; #[async_trait] impl StatusDecoder for DefaultStatusDecoder { @@ -51,7 +51,7 @@ impl StatusDecoder for DefaultStatusDecoder { } } -pub fn default_decode_exit_code(status: ExitStatus) -> i32 { +fn default_decode_exit_code(status: ExitStatus) -> i32 { let exit_code; #[cfg(unix)] @@ -70,11 +70,12 @@ pub fn default_decode_exit_code(status: ExitStatus) -> i32 { exit_code.unwrap_or(-1) } -pub struct MiniperfStatusDecoder { +pub(crate) struct MiniperfStatusDecoder { out_path: AbsNormPathBuf, } impl MiniperfStatusDecoder { + #[allow(dead_code)] pub fn new(out_path: AbsNormPathBuf) -> Self { Self { out_path } } diff --git a/app/buck2_forkserver/src/unix/mod.rs b/app/buck2_forkserver/src/unix/mod.rs index ce0e3589e46a2..e783bbd78909a 100644 --- a/app/buck2_forkserver/src/unix/mod.rs +++ b/app/buck2_forkserver/src/unix/mod.rs @@ -9,7 +9,7 @@ mod command; mod launch; -pub mod process_group; +pub(crate) mod process_group; mod service; pub use command::run_forkserver; diff --git a/app/buck2_forkserver/src/unix/process_group.rs b/app/buck2_forkserver/src/unix/process_group.rs index 07bae73ed6b85..811049b7c265d 100644 --- a/app/buck2_forkserver/src/unix/process_group.rs +++ b/app/buck2_forkserver/src/unix/process_group.rs @@ -7,6 +7,10 @@ * of this source tree. */ +use std::os::unix::process::CommandExt; +use std::process::Command as StdCommand; +use std::process::ExitStatus; +use std::process::Stdio; use std::time::Duration; use anyhow::Context; @@ -14,23 +18,65 @@ use buck2_common::kill_util::try_terminate_process_gracefully; use nix::sys::signal; use nix::sys::signal::Signal; use nix::unistd::Pid; +use tokio::io; use tokio::process::Child; +use tokio::process::ChildStderr; +use tokio::process::ChildStdout; +use tokio::process::Command; -pub struct ProcessGroupImpl { +pub(crate) struct ProcessCommandImpl { + inner: Command, +} + +impl ProcessCommandImpl { + pub(crate) fn new(mut cmd: StdCommand) -> Self { + cmd.process_group(0); + Self { inner: cmd.into() } + } + + pub(crate) fn spawn(&mut self) -> io::Result { + self.inner.spawn() + } + + pub(crate) fn stdout(&mut self, stdout: Stdio) { + self.inner.stdout(stdout); + } + + pub(crate) fn stderr(&mut self, stdout: Stdio) { + self.inner.stderr(stdout); + } +} + +pub(crate) struct ProcessGroupImpl { inner: Child, } impl ProcessGroupImpl { - pub fn new(child: Child) -> anyhow::Result { + pub(crate) fn new(child: Child) -> anyhow::Result { Ok(ProcessGroupImpl { inner: child }) } - pub fn child(&mut self) -> &mut Child { - &mut self.inner + pub(crate) fn take_stdout(&mut self) -> Option { + self.inner.stdout.take() + } + + pub(crate) fn take_stderr(&mut self) -> Option { + self.inner.stderr.take() + } + + pub(crate) async fn wait(&mut self) -> io::Result { + self.inner.wait().await + } + + pub(crate) fn id(&self) -> Option { + self.inner.id() } // On unix we use killpg to kill the whole process tree - pub async fn kill(&self, graceful_shutdown_timeout_s: Option) -> anyhow::Result<()> { + pub(crate) async fn kill( + &self, + graceful_shutdown_timeout_s: Option, + ) -> anyhow::Result<()> { let pid: i32 = self .inner .id() diff --git a/app/buck2_forkserver/src/unix/service.rs b/app/buck2_forkserver/src/unix/service.rs index a8d68426032cc..111559cc02aab 100644 --- a/app/buck2_forkserver/src/unix/service.rs +++ b/app/buck2_forkserver/src/unix/service.rs @@ -44,8 +44,7 @@ use tonic::Streaming; use crate::convert::encode_event_stream; use crate::run::maybe_absolutize_exe; -use crate::run::prepare_command; -use crate::run::process_group::ProcessGroup; +use crate::run::process_group::ProcessCommand; use crate::run::status_decoder::DefaultStatusDecoder; use crate::run::status_decoder::MiniperfStatusDecoder; use crate::run::stream_command_events; @@ -161,16 +160,14 @@ impl Forkserver for UnixForkserverService { } } - let mut cmd = prepare_command(cmd); let stream_stdio = std_redirects.is_none(); + let mut cmd = ProcessCommand::new(cmd); if let Some(std_redirects) = std_redirects { cmd.stdout(File::create(OsStr::from_bytes(&std_redirects.stdout))?); cmd.stderr(File::create(OsStr::from_bytes(&std_redirects.stderr))?); } - let child = cmd.spawn(); - let process_group = child - .map_err(anyhow::Error::from) - .and_then(ProcessGroup::new); + + let process_group = cmd.spawn().map_err(anyhow::Error::from); let timeout = timeout_into_cancellation(timeout); diff --git a/app/buck2_forkserver/src/win/child_process.rs b/app/buck2_forkserver/src/win/child_process.rs new file mode 100644 index 0000000000000..7149f20a29e07 --- /dev/null +++ b/app/buck2_forkserver/src/win/child_process.rs @@ -0,0 +1,146 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::future::Future; +use std::os::windows::io::AsRawHandle; +use std::pin::Pin; +use std::process::Child; +use std::process::ExitStatus; +use std::task::Context; +use std::task::Poll; + +use tokio::io; +use tokio::sync::oneshot; +use winapi::um::handleapi; +use winapi::um::threadpoollegacyapiset::UnregisterWaitEx; +use winapi::um::winbase; +use winapi::um::winnt; +use winapi::um::winnt::HANDLE; + +pub(crate) struct ChildProcess { + inner: Child, + waiting: Option, +} + +impl ChildProcess { + pub(crate) fn new(child: Child) -> Self { + Self { + inner: child, + waiting: None, + } + } + + pub(crate) fn as_std(&self) -> &Child { + &self.inner + } + + pub(crate) fn as_std_mut(&mut self) -> &mut Child { + &mut self.inner + } +} + +// This implementation is a copy of tokio internal Future implementation on their Child. +// See https://github.com/tokio-rs/tokio/blob/master/tokio/src/process/windows.rs#L102 +impl Future for ChildProcess { + type Output = io::Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let inner = Pin::get_mut(self); + loop { + if let Some(ref mut w) = inner.waiting { + match Pin::new(&mut w.rx).poll(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(e)) => Err(io::Error::new(io::ErrorKind::Other, e))?, + Poll::Pending => return Poll::Pending, + } + let status = inner.inner.try_wait()?.ok_or(io::Error::new( + io::ErrorKind::Other, + "exit status is not available", + ))?; + return Poll::Ready(Ok(status)); + } + + if let Some(e) = inner.inner.try_wait()? { + return Poll::Ready(Ok(e)); + } + let (tx, rx) = oneshot::channel(); + let ptr = Box::into_raw(Box::new(Some(tx))); + let mut wait_object = handleapi::INVALID_HANDLE_VALUE; + let rc = unsafe { + winbase::RegisterWaitForSingleObject( + &mut wait_object, + inner.inner.as_raw_handle(), + Some(callback), + ptr as *mut _, + winbase::INFINITE, + winnt::WT_EXECUTEINWAITTHREAD | winnt::WT_EXECUTEONLYONCE, + ) + }; + if rc == 0 { + let err = io::Error::last_os_error(); + drop(unsafe { Box::from_raw(ptr) }); + return Poll::Ready(Err(err)); + } + inner.waiting = Some(Waiting { + rx, + wait_object, + tx: ptr, + }); + } + } +} + +// Waiting for a signal from a `wait_object` handle +struct Waiting { + rx: oneshot::Receiver<()>, + wait_object: HANDLE, + // we're using raw pointer to pass it through ffi to callback + tx: *mut Option>, +} + +unsafe impl Sync for Waiting {} +unsafe impl Send for Waiting {} + +impl Drop for Waiting { + fn drop(&mut self) { + unsafe { + let rc = UnregisterWaitEx(self.wait_object, handleapi::INVALID_HANDLE_VALUE); + if rc == 0 { + panic!("failed to unregister: {}", io::Error::last_os_error()); + } + drop(Box::from_raw(self.tx)); + } + } +} + +unsafe extern "system" fn callback(ptr: *mut std::ffi::c_void, _timer_fired: winnt::BOOLEAN) { + let complete = &mut *(ptr as *mut Option>); + // ignore the error as Waiting could be drop before the callback fires + let _ = complete.take().unwrap().send(()); +} + +#[cfg(test)] +mod tests { + use buck2_util::process::background_command; + + use crate::win::child_process::ChildProcess; + + #[tokio::test] + async fn test_child_process() -> anyhow::Result<()> { + let mut cmd = background_command("cmd"); + let cmd = cmd.arg("/c").arg("exit 2"); + + let child = cmd.spawn().unwrap(); + let proc = ChildProcess::new(child); + + let status = proc.await?; + assert_eq!(status.code(), Some(2)); + Ok(()) + } +} diff --git a/app/buck2_forkserver/src/win/job_object.rs b/app/buck2_forkserver/src/win/job_object.rs index db30397021c9e..f0cece8c11306 100644 --- a/app/buck2_forkserver/src/win/job_object.rs +++ b/app/buck2_forkserver/src/win/job_object.rs @@ -15,12 +15,12 @@ use winapi::shared::minwindef::FALSE; use winapi::um::jobapi2; use winapi::um::winnt::HANDLE; -pub struct JobObject { +pub(crate) struct JobObject { handle: WinapiHandle, } impl JobObject { - pub fn new() -> anyhow::Result { + pub(crate) fn new() -> anyhow::Result { let handle = unsafe { let handle = jobapi2::CreateJobObjectW(ptr::null_mut(), ptr::null_mut()); WinapiHandle::new(handle) @@ -31,7 +31,7 @@ impl JobObject { Ok(Self { handle }) } - pub fn assign_process(&self, process: HANDLE) -> anyhow::Result<()> { + pub(crate) fn assign_process(&self, process: HANDLE) -> anyhow::Result<()> { let res = unsafe { jobapi2::AssignProcessToJobObject(self.handle.handle(), process) }; if res == FALSE { Err(anyhow::anyhow!(io::Error::last_os_error())) @@ -40,7 +40,7 @@ impl JobObject { } } - pub fn terminate(&self, exit_code: u32) -> anyhow::Result<()> { + pub(crate) fn terminate(&self, exit_code: u32) -> anyhow::Result<()> { let res = unsafe { jobapi2::TerminateJobObject(self.handle.handle(), exit_code) }; if res == FALSE { return Err(anyhow::anyhow!(io::Error::last_os_error())); diff --git a/app/buck2_forkserver/src/win/mod.rs b/app/buck2_forkserver/src/win/mod.rs index 5b48c230b726a..67c8935d7943b 100644 --- a/app/buck2_forkserver/src/win/mod.rs +++ b/app/buck2_forkserver/src/win/mod.rs @@ -7,5 +7,6 @@ * of this source tree. */ -pub mod job_object; -pub mod process_group; +pub(crate) mod child_process; +pub(crate) mod job_object; +pub(crate) mod process_group; diff --git a/app/buck2_forkserver/src/win/process_group.rs b/app/buck2_forkserver/src/win/process_group.rs index d76c3f8823fdc..1324c33609389 100644 --- a/app/buck2_forkserver/src/win/process_group.rs +++ b/app/buck2_forkserver/src/win/process_group.rs @@ -7,107 +7,158 @@ * of this source tree. */ -use buck2_wrapper_common::winapi_handle::WinapiHandle; -use tokio::process::Child; +use std::os::windows::io::AsRawHandle; +use std::os::windows::process::ChildExt; +use std::os::windows::process::CommandExt; +use std::process::Child; +use std::process::Command; +use std::process::ExitStatus; +use std::process::Stdio; + +use buck2_error::Context; +use tokio::io; +use tokio::process::ChildStderr; +use tokio::process::ChildStdout; use winapi::shared::minwindef; -use winapi::um::handleapi; use winapi::um::processthreadsapi; -use winapi::um::tlhelp32; -use winapi::um::winnt; +use crate::win::child_process::ChildProcess; use crate::win::job_object::JobObject; -pub struct ProcessGroupImpl { - inner: Child, +pub(crate) struct ProcessCommandImpl { + inner: Command, +} + +impl ProcessCommandImpl { + pub(crate) fn new(mut cmd: Command) -> Self { + // On windows we create suspended process to assign it to a job (group) and then resume. + // This is necessary because the process might finish before we add it to a job + cmd.creation_flags( + winapi::um::winbase::CREATE_NO_WINDOW | winapi::um::winbase::CREATE_SUSPENDED, + ); + Self { inner: cmd } + } + + pub(crate) fn spawn(&mut self) -> io::Result { + self.inner.spawn() + } + + #[allow(dead_code)] + pub(crate) fn stdout(&mut self, stdout: Stdio) { + self.inner.stdout(stdout); + } + + #[allow(dead_code)] + pub(crate) fn stderr(&mut self, stdout: Stdio) { + self.inner.stderr(stdout); + } +} + +/// Keeps track of the exit status of a child process without worrying about +/// polling the underlying futures even after they have completed. +enum FusedChild { + Child(ChildProcess), + Done(ExitStatus), +} + +impl FusedChild { + fn as_option(&self) -> Option<&ChildProcess> { + match &self { + FusedChild::Child(child) => Some(child), + FusedChild::Done(_) => None, + } + } + + fn as_option_mut(&mut self) -> Option<&mut ChildProcess> { + match self { + FusedChild::Child(child) => Some(child), + FusedChild::Done(_) => None, + } + } +} + +pub(crate) struct ProcessGroupImpl { + child: FusedChild, job: JobObject, } impl ProcessGroupImpl { - pub fn new(child: Child) -> anyhow::Result { + pub(crate) fn new(child: Child) -> anyhow::Result { let job = JobObject::new()?; - if let Some(handle) = child.raw_handle() { - job.assign_process(handle)?; - } - let process = ProcessGroupImpl { inner: child, job }; + job.assign_process(child.as_raw_handle())?; + let process = ProcessGroupImpl { + child: FusedChild::Child(ChildProcess::new(child)), + job, + }; // We create suspended process to assign it to a job (group) // So we resume the process after assignment process.resume()?; Ok(process) } - pub fn child(&mut self) -> &mut Child { - &mut self.inner + pub(crate) fn take_stdout(&mut self) -> Option { + self.child + .as_option_mut()? + .as_std_mut() + .stdout + .take() + .and_then(|s| ChildStdout::from_std(s).ok()) } - // On Windows we use JobObject API to kill the whole process tree - pub async fn kill(&self, _graceful_shutdown_timeout_s: Option) -> anyhow::Result<()> { - self.job.terminate(0) + pub(crate) fn take_stderr(&mut self) -> Option { + self.child + .as_option_mut()? + .as_std_mut() + .stderr + .take() + .and_then(|s| ChildStderr::from_std(s).ok()) } - fn resume(&self) -> anyhow::Result<()> { - let process_id = self - .inner - .id() - .ok_or_else(|| anyhow::anyhow!("Failed to get the process id"))?; - unsafe { - let main_thread_id = get_main_thread(process_id)?; - resume_thread(main_thread_id) - } - } -} + pub(crate) async fn wait(&mut self) -> io::Result { + match &mut self.child { + FusedChild::Done(exit) => Ok(*exit), + FusedChild::Child(child) => { + // Ensure stdin is closed so the child isn't stuck waiting on + // input while the parent is waiting for it to exit. + drop(child.as_std_mut().stdin.take()); + let ret = child.await; -// We need main thread handle to resume process after assigning it to JobObject -// Currently there is no way to get main thread handle from tokio's process::Child -// We use CreateToolhelp32Snapshot to get a snapshot of all threads in the system -// and find the one with the given process_id. -// todo(yurysamkevich): use main_thread_handle once issue is closed -// https://github.com/tokio-rs/tokio/issues/6153 -unsafe fn get_main_thread(process_id: u32) -> anyhow::Result { - let snapshot_handle = tlhelp32::CreateToolhelp32Snapshot(tlhelp32::TH32CS_SNAPTHREAD, 0); - if snapshot_handle == handleapi::INVALID_HANDLE_VALUE { - return Err(anyhow::anyhow!("Failed to list threads")); - } - let snapshot_handle = WinapiHandle::new(snapshot_handle); - let mut thread_entry_32 = tlhelp32::THREADENTRY32 { - dwSize: std::mem::size_of::() as u32, - cntUsage: 0, - th32ThreadID: 0, - th32OwnerProcessID: 0, - tpBasePri: 0, - tpDeltaPri: 0, - dwFlags: 0, - }; - let raw_pointer_to_thread_entry_32 = &mut thread_entry_32 as *mut tlhelp32::THREADENTRY32; - - let mut thread_result = - tlhelp32::Thread32First(snapshot_handle.handle(), raw_pointer_to_thread_entry_32); - while thread_result == minwindef::TRUE { - if thread_entry_32.dwSize as usize - >= std::mem::offset_of!(tlhelp32::THREADENTRY32, th32OwnerProcessID) - { - if thread_entry_32.th32OwnerProcessID == process_id { - return Ok(thread_entry_32.th32ThreadID); + if let Ok(exit) = ret { + self.child = FusedChild::Done(exit); + } + + ret } } - thread_result = winapi::um::tlhelp32::Thread32Next( - snapshot_handle.handle(), - raw_pointer_to_thread_entry_32, - ); } - Err(anyhow::anyhow!("Failed to find thread to resume")) -} -unsafe fn resume_thread(thread_id: minwindef::DWORD) -> anyhow::Result<()> { - let thread_handle = - processthreadsapi::OpenThread(winnt::THREAD_SUSPEND_RESUME, minwindef::FALSE, thread_id); - if thread_handle.is_null() { - return Err(anyhow::anyhow!("Failed to open thread to resume")); + pub(crate) fn id(&self) -> Option { + Some(self.child.as_option()?.as_std().id()) + } + + // On Windows we use JobObject API to kill the whole process tree + pub(crate) async fn kill( + &self, + _graceful_shutdown_timeout_s: Option, + ) -> anyhow::Result<()> { + self.job.terminate(0) } - let thread_handle = WinapiHandle::new(thread_handle); - let resume_thread_result = processthreadsapi::ResumeThread(thread_handle.handle()); - if resume_thread_result == minwindef::DWORD::MAX { - Err(anyhow::anyhow!("Failed to resume thread")) - } else { - Ok(()) + + fn resume(&self) -> anyhow::Result<()> { + let handle = self + .child + .as_option() + .context("can't resume an exited process")? + .as_std() + .main_thread_handle() + .as_raw_handle(); + unsafe { + let resume_thread_result = processthreadsapi::ResumeThread(handle); + if resume_thread_result == minwindef::DWORD::MAX { + Err(anyhow::anyhow!("Failed to resume thread")) + } else { + Ok(()) + } + } } } diff --git a/app/buck2_futures/src/cancellation/future.rs b/app/buck2_futures/src/cancellation/future.rs index 0ad52850c1fb8..017c551dbf16e 100644 --- a/app/buck2_futures/src/cancellation/future.rs +++ b/app/buck2_futures/src/cancellation/future.rs @@ -12,7 +12,6 @@ //! This future is intended to be spawned on tokio-runtime directly, and for its results to be //! accessed via the joinhandle. //! It is not intended to be polled directly. -//! use std::future::Future; use std::mem; diff --git a/app/buck2_futures/src/owning_future.rs b/app/buck2_futures/src/owning_future.rs index 1087892d4aaa7..4b74a1d60f2f7 100644 --- a/app/buck2_futures/src/owning_future.rs +++ b/app/buck2_futures/src/owning_future.rs @@ -12,7 +12,6 @@ //! This future is intended to be spawned on tokio-runtime directly, and for its results to be //! accessed via the joinhandle. //! It is not intended to be polled directly. -//! use std::future::Future; use std::marker::PhantomPinned; diff --git a/app/buck2_futures/src/spawn.rs b/app/buck2_futures/src/spawn.rs index dd80b6df8e4af..1f3d8080a781c 100644 --- a/app/buck2_futures/src/spawn.rs +++ b/app/buck2_futures/src/spawn.rs @@ -9,7 +9,6 @@ //! The future that is spawned, but has various more strict cancellation behaviour than //! tokio's JoinHandle -//! use std::any::Any; use std::pin::Pin; diff --git a/app/buck2_http/src/client/builder.rs b/app/buck2_http/src/client/builder.rs index d5d2f60bb07ee..5f0f7395330bf 100644 --- a/app/buck2_http/src/client/builder.rs +++ b/app/buck2_http/src/client/builder.rs @@ -61,6 +61,7 @@ pub struct HttpClientBuilder { proxies: Vec, max_redirects: Option, supports_vpnless: bool, + http2: bool, timeout_config: Option, } @@ -98,6 +99,7 @@ impl HttpClientBuilder { proxies: Vec::new(), max_redirects: None, supports_vpnless: false, + http2: true, timeout_config: None, }) } @@ -197,6 +199,11 @@ impl HttpClientBuilder { self } + pub fn with_http2(&mut self, http2: bool) -> &mut Self { + self.http2 = http2; + self + } + pub fn supports_vpnless(&self) -> bool { self.supports_vpnless } @@ -244,7 +251,7 @@ impl HttpClientBuilder { // Proxied http client with TLS. (proxies @ [_, ..], Some(timeout_config)) => { - let https_connector = build_https_connector(self.tls_config.clone()); + let https_connector = build_https_connector(self.tls_config.clone(), self.http2); let timeout_connector = timeout_config.to_connector(https_connector); // Re-use TLS config from https connection for communication with proxies. let proxy_connector = build_proxy_connector( @@ -255,7 +262,7 @@ impl HttpClientBuilder { Arc::new(hyper::Client::builder().build::<_, Body>(proxy_connector)) } (proxies @ [_, ..], None) => { - let https_connector = build_https_connector(self.tls_config.clone()); + let https_connector = build_https_connector(self.tls_config.clone(), self.http2); let proxy_connector = build_proxy_connector(proxies, https_connector, Some(self.tls_config.clone())); Arc::new(hyper::Client::builder().build::<_, Body>(proxy_connector)) @@ -263,12 +270,12 @@ impl HttpClientBuilder { // Client with TLS only. ([], Some(timeout_config)) => { - let https_connector = build_https_connector(self.tls_config.clone()); + let https_connector = build_https_connector(self.tls_config.clone(), self.http2); let timeout_connector = timeout_config.to_connector(https_connector); Arc::new(hyper::Client::builder().build::<_, Body>(timeout_connector)) } ([], None) => { - let https_connector = build_https_connector(self.tls_config.clone()); + let https_connector = build_https_connector(self.tls_config.clone(), self.http2); Arc::new(hyper::Client::builder().build::<_, Body>(https_connector)) } } @@ -279,18 +286,23 @@ impl HttpClientBuilder { inner: self.build_inner(), max_redirects: self.max_redirects, supports_vpnless: self.supports_vpnless, + http2: self.http2, stats: HttpNetworkStats::new(), } } } -fn build_https_connector(tls_config: ClientConfig) -> HttpsConnector { - HttpsConnectorBuilder::new() +fn build_https_connector(tls_config: ClientConfig, http2: bool) -> HttpsConnector { + let builder = HttpsConnectorBuilder::new() .with_tls_config(tls_config) .https_or_http() - .enable_http1() - .enable_http2() - .build() + .enable_http1(); + + if http2 { + builder.enable_http2().build() + } else { + builder.build() + } } /// Build a proxy connector using `proxies`, wrapping underlying `connector`, @@ -357,6 +369,16 @@ mod tests { Ok(()) } + #[test] + fn test_http2_option() -> anyhow::Result<()> { + let mut builder = HttpClientBuilder::https_with_system_roots()?; + assert!(builder.http2); + builder.with_http2(false); + + assert!(!builder.http2); + Ok(()) + } + #[test] fn test_with_max_redirects_overrides_default() -> anyhow::Result<()> { let mut builder = HttpClientBuilder::https_with_system_roots()?; diff --git a/app/buck2_http/src/client/mod.rs b/app/buck2_http/src/client/mod.rs index db88db8b07269..f1fb0bac5b475 100644 --- a/app/buck2_http/src/client/mod.rs +++ b/app/buck2_http/src/client/mod.rs @@ -47,6 +47,7 @@ pub struct HttpClient { inner: Arc, max_redirects: Option, supports_vpnless: bool, + http2: bool, stats: HttpNetworkStats, } @@ -192,6 +193,10 @@ impl HttpClient { pub fn supports_vpnless(&self) -> bool { self.supports_vpnless } + + pub fn http2(&self) -> bool { + self.http2 + } } /// Trait wrapper around a hyper::Client because hyper::Client is parameterized by diff --git a/app/buck2_http/src/lib.rs b/app/buck2_http/src/lib.rs index 0765df9aed41f..60336ab4f2c07 100644 --- a/app/buck2_http/src/lib.rs +++ b/app/buck2_http/src/lib.rs @@ -45,6 +45,7 @@ fn category_from_status(status: StatusCode) -> Option { } #[derive(Debug, buck2_error::Error)] +#[buck2(tag = Http)] pub enum HttpError { #[error("HTTP URI Error: URI {uri} is malformed: {source:?}")] InvalidUri { diff --git a/app/buck2_http/src/retries.rs b/app/buck2_http/src/retries.rs index 464ee2b57325f..15c86fd279022 100644 --- a/app/buck2_http/src/retries.rs +++ b/app/buck2_http/src/retries.rs @@ -54,7 +54,7 @@ pub trait AsHttpError { pub async fn http_retry(exec: Exec, mut intervals: Vec) -> Result where Exec: Fn() -> F, - E: AsHttpError + std::fmt::Display, + E: std::error::Error + AsHttpError + std::fmt::Display + Send + Sync + 'static, F: Future>, { intervals.insert(0, Duration::from_secs(0)); @@ -63,24 +63,26 @@ where while let Some(duration) = backoff.next() { tokio::time::sleep(duration).await; - let res = exec().await; + let err = match exec().await { + Ok(val) => return Ok(val), + Err(err) => err, + }; - let http_error = res.as_ref().err().and_then(|err| err.as_http_error()); - - if let Some(http_error) = http_error { + if let Some(http_error) = err.as_http_error() { if http_error.is_retryable() { if let Some(b) = backoff.peek() { tracing::warn!( "Retrying a HTTP error after {} seconds: {:#}", b.as_secs(), - http_error + // Print as a buck2_error to make sure we get the source + buck2_error::Error::from(err) ); continue; } } } - return res; + return Err(err); } unreachable!("The loop above will exit before we get to the end") diff --git a/app/buck2_install_proto/BUCK b/app/buck2_install_proto/BUCK index c1d217becc7b4..d11b5c70178fc 100644 --- a/app/buck2_install_proto/BUCK +++ b/app/buck2_install_proto/BUCK @@ -1,6 +1,5 @@ load("@fbcode//buck2:proto_defs.bzl", "rust_protobuf_library") load("@fbcode//grpc_fb/codegen:buck_macros.bzl", "grpc_library") -load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") load("@fbsource//tools/build_defs:glob_defs.bzl", "glob") oncall("build_infra") @@ -15,19 +14,6 @@ rust_protobuf_library( ], ) -python_binary( - # @autodeps-skip - name = "installer", - srcs = [ - "main.py", - ], - base_module = "", - main_function = "main.main", - deps = [ - ":install-py", - ], -) - grpc_library( name = "install", srcs = [ diff --git a/app/buck2_install_proto/main.py b/app/buck2_install_proto/main.py deleted file mode 100644 index b123deacfe7a4..0000000000000 --- a/app/buck2_install_proto/main.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under both the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree and the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. - -import argparse -import os -import signal -import subprocess -import sys -import threading -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from typing import Optional - -import grpc -from buck2.app.buck2_install_proto import install_pb2, install_pb2_grpc - - -class RsyncInstallerService(install_pb2_grpc.InstallerServicer): - def __init__(self, stop_event, argsparse, *args, **kwargs): - self.args = argsparse - self.stop_event = stop_event - if argsparse.install_location == "": - self.dst = argsparse.dst - else: - self.dst = f"{argsparse.install_location}:{argsparse.dst}" - - def Install(self, request, _context): - install_id = request.install_id - files = request.files - - print( - f"Received request with install info: {install_id=:} and {len(files)} files" - ) - - install_response = install_pb2.InstallResponse() - install_response.install_id = install_id - return install_response - - def FileReady(self, request, _context): - (_out, stderr, code) = self.rsync_install( - request.path, os.path.join(self.dst, request.name) - ) - response = { - "install_id": request.install_id, - "name": f"{request.name}", - "path": request.path, - } - - if code != 0: - error_detail = install_pb2.ErrorDetail() - error_detail.message = stderr - response["error_detail"] = error_detail - - file_response = install_pb2.FileResponse(**response) - return file_response - - def ShutdownServer(self, _request, _context): - shutdown(self.stop_event) - response = install_pb2.ShutdownResponse() - return response - - def rsync_install(self, src, dst): - if not (dst_parent := Path(dst).parent).exists(): - dst_parent.mkdir(parents=True, exist_ok=True) - cmd = [ - "rsync", - "-aL", - str(src), - str(dst), - ] - cp = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8" - ) - stdout, stderr = cp.communicate() - code = cp.returncode - return (stdout, stderr, code) - - -def try_command( - cmd: [str], - err_msg: str, - cwd: Optional = None, - env: Optional = None, - shell: bool = False, -): - try: - output = subprocess.check_output(cmd, cwd=cwd, env=env, shell=shell) - return output - except Exception as e: - print(f"Failed step {err_msg} with {str(e)}") - raise e - - -def shutdown(stop_event): - stop_event.set() - - -def serve(args): - print(f"Starting installer server installing to {args.dst}") - server = grpc.server( - thread_pool=ThreadPoolExecutor(max_workers=50), - options=[("grpc.max_receive_message_length", 500 * 1024 * 1024)], - ) - stop_event = threading.Event() - install_pb2_grpc.add_InstallerServicer_to_server( - RsyncInstallerService(stop_event, args), server - ) - ## https://grpc.github.io/grpc/python/grpc.html - listen_addr = server.add_insecure_port("[::]:" + args.tcp_port) - print(f"Started server on {listen_addr} w/ pid {os.getpid()}") - server.start() - signal.signal(signal.SIGINT, lambda x, y: shutdown(stop_event)) - try: - stop_event.wait() - print("Stopped RPC server, Waiting for RPCs to complete...") - server.stop(1).wait() - finally: - print("Exiting installer") - - -def parse_args(args=None): - parser = argparse.ArgumentParser(description="Parse args for install location") - parser.add_argument( - "--install-location", - help="Defines install hostname (I.E. devserver)", - default="", - ) - parser.add_argument( - "--dst", - type=str, - help="destination rsync target folder", - default="/tmp/buck2install/", - ) - parser.add_argument( - "--tcp-port", - type=str, - help="tcp port for installer to connect to", - required=True, - ) - # no need to parse --tcp-port and other not related params - args, _ = parser.parse_known_args(args) - return args - - -def main() -> None: - args = parse_args(sys.argv[1:]) - serve(args) - - -if __name__ == "__main__": - main() # pragma: no cover diff --git a/app/buck2_interpreter/src/dice/starlark_provider.rs b/app/buck2_interpreter/src/dice/starlark_provider.rs index 16e198dfcd450..080f289d03129 100644 --- a/app/buck2_interpreter/src/dice/starlark_provider.rs +++ b/app/buck2_interpreter/src/dice/starlark_provider.rs @@ -9,6 +9,8 @@ use std::ops::Deref; +use buck2_common::legacy_configs::dice::HasLegacyConfigs; +use buck2_common::legacy_configs::view::LegacyBuckConfigView; use dice::DiceComputations; use starlark::environment::FrozenModule; use starlark::environment::Module; @@ -36,6 +38,11 @@ pub async fn with_starlark_eval_provider, R> description: String, closure: impl FnOnce(&mut dyn StarlarkEvaluatorProvider, D) -> anyhow::Result, ) -> anyhow::Result { + let root_buckconfig = ctx.get_legacy_root_config_on_dice().await?; + let root_buckconfig_view: &dyn LegacyBuckConfigView = &root_buckconfig; + let starlark_max_callstack_size = + root_buckconfig_view.parse::("buck2", "starlark_max_callstack_size")?; + let debugger_handle = ctx.get_starlark_debugger_handle(); let debugger = match debugger_handle { Some(v) => Some(v.start_eval(&description).await?), @@ -45,11 +52,16 @@ pub async fn with_starlark_eval_provider, R> struct EvalProvider<'a, 'b> { profiler: &'a mut StarlarkProfilerOrInstrumentation<'b>, debugger: Option>, + starlark_max_callstack_size: Option, } impl StarlarkEvaluatorProvider for EvalProvider<'_, '_> { fn make<'v, 'a>(&mut self, module: &'v Module) -> anyhow::Result> { let mut eval = Evaluator::new(module); + if let Some(stack_size) = self.starlark_max_callstack_size { + eval.set_max_callstack_size(stack_size)?; + } + self.profiler.initialize(&mut eval)?; if let Some(v) = &mut self.debugger { v.initialize(&mut eval)?; @@ -70,6 +82,7 @@ pub async fn with_starlark_eval_provider, R> let mut provider = EvalProvider { profiler: profiler_instrumentation, debugger, + starlark_max_callstack_size, }; // If we're debugging, we need to move this to a tokio blocking task. diff --git a/app/buck2_interpreter/src/error.rs b/app/buck2_interpreter/src/error.rs index 1d8d38c4cd7a9..6e606329159f3 100644 --- a/app/buck2_interpreter/src/error.rs +++ b/app/buck2_interpreter/src/error.rs @@ -60,6 +60,7 @@ impl std::error::Error for BuckStarlarkError { | starlark::ErrorKind::Value(_) => Some(buck2_error::Category::User), starlark::ErrorKind::Function(_) => Some(buck2_error::Category::User), starlark::ErrorKind::Scope(_) => Some(buck2_error::Category::User), + starlark::ErrorKind::Lexer(_) => Some(buck2_error::Category::User), _ => None, }; let tags = match self.e.kind() { @@ -72,6 +73,7 @@ impl std::error::Error for BuckStarlarkError { starlark::ErrorKind::Value(_) => "BuckStarlarkError::Value", starlark::ErrorKind::Function(_) => "BuckStarlarkError::Function", starlark::ErrorKind::Scope(_) => "BuckStarlarkError::Scope", + starlark::ErrorKind::Lexer(_) => "BuckStarlarkError::Lexer", _ => "BuckStarlarkError", }; buck2_error::provide_metadata( diff --git a/app/buck2_interpreter_for_build/src/attrs/attrs_global.rs b/app/buck2_interpreter_for_build/src/attrs/attrs_global.rs index 29ffcda6c7b0e..6f10264b445a2 100644 --- a/app/buck2_interpreter_for_build/src/attrs/attrs_global.rs +++ b/app/buck2_interpreter_for_build/src/attrs/attrs_global.rs @@ -651,7 +651,7 @@ pub fn register_attrs(globals: &mut GlobalsBuilder) { } #[cfg(test)] -mod test { +mod tests { use super::*; use crate::interpreter::testing::Tester; diff --git a/app/buck2_interpreter_for_build/src/attrs/coerce/coerced_attr.rs b/app/buck2_interpreter_for_build/src/attrs/coerce/coerced_attr.rs index a344c6d16d2ae..d801f962c52a3 100644 --- a/app/buck2_interpreter_for_build/src/attrs/coerce/coerced_attr.rs +++ b/app/buck2_interpreter_for_build/src/attrs/coerce/coerced_attr.rs @@ -30,7 +30,7 @@ enum SelectError { ValueNotDict(String), #[error("addition not supported for this attribute type `{0}`, got `{1}`.")] ConcatNotSupported(String, String), - #[error("select() cannot be used in non-configuable attribute")] + #[error("select() cannot be used in non-configurable attribute")] SelectCannotBeUsedForNonConfigurableAttr, #[error("duplicate `\"DEFAULT\"` key in `select()` (internal error)")] DuplicateDefaultKey, @@ -39,7 +39,7 @@ enum SelectError { pub trait CoercedAttrExr: Sized { fn coerce( attr: &AttrType, - configuable: AttrIsConfigurable, + configurable: AttrIsConfigurable, ctx: &dyn AttrCoercionContext, value: Value, default_attr: Option<&Self>, @@ -49,7 +49,7 @@ pub trait CoercedAttrExr: Sized { impl CoercedAttrExr for CoercedAttr { fn coerce( attr: &AttrType, - configuable: AttrIsConfigurable, + configurable: AttrIsConfigurable, ctx: &dyn AttrCoercionContext, value: Value, default_attr: Option<&Self>, @@ -66,7 +66,7 @@ impl CoercedAttrExr for CoercedAttr { // are actually compatible (i.e. selectable can ensure that both sides are // lists, we can ensure that both sides are List) if let Some(selector) = StarlarkSelector::from_value(value) { - if let AttrIsConfigurable::No = configuable { + if let AttrIsConfigurable::No = configurable { return Err(SelectError::SelectCannotBeUsedForNonConfigurableAttr.into()); } @@ -84,7 +84,7 @@ impl CoercedAttrExr for CoercedAttr { })?; let v = match default_attr { Some(default_attr) if v.is_none() => default_attr.clone(), - _ => CoercedAttr::coerce(attr, configuable, ctx, v, None)?, + _ => CoercedAttr::coerce(attr, configurable, ctx, v, None)?, }; if k == "DEFAULT" { if default.is_some() { @@ -114,12 +114,12 @@ impl CoercedAttrExr for CoercedAttr { format!("{} + {}", l, r) ))); } - let l = CoercedAttr::coerce(attr, configuable, ctx, l, None)?; + let l = CoercedAttr::coerce(attr, configurable, ctx, l, None)?; let mut l = match l { CoercedAttr::Concat(l) => l.into_vec(), l => vec![l], }; - let r = CoercedAttr::coerce(attr, configuable, ctx, r, None)?; + let r = CoercedAttr::coerce(attr, configurable, ctx, r, None)?; let r = match r { CoercedAttr::Concat(r) => r.into_vec(), r => vec![r], @@ -131,7 +131,7 @@ impl CoercedAttrExr for CoercedAttr { } } else { Ok(attr - .coerce_item(configuable, ctx, value) + .coerce_item(configurable, ctx, value) .with_context(|| format!("Error coercing {}", value))?) } } diff --git a/app/buck2_interpreter_for_build/src/interpreter/dice_calculation_delegate.rs b/app/buck2_interpreter_for_build/src/interpreter/dice_calculation_delegate.rs index 271da242e6732..ddbf17b030dbd 100644 --- a/app/buck2_interpreter_for_build/src/interpreter/dice_calculation_delegate.rs +++ b/app/buck2_interpreter_for_build/src/interpreter/dice_calculation_delegate.rs @@ -30,6 +30,7 @@ use buck2_events::dispatch::span; use buck2_events::dispatch::span_async; use buck2_futures::cancellation::CancellationContext; use buck2_interpreter::dice::starlark_provider::with_starlark_eval_provider; +use buck2_interpreter::error::BuckStarlarkError; use buck2_interpreter::file_loader::LoadedModule; use buck2_interpreter::file_loader::ModuleDeps; use buck2_interpreter::import_paths::HasImportPaths; @@ -45,8 +46,12 @@ use dice::DiceComputations; use dice::Key; use dupe::Dupe; use futures::future; +use indoc::indoc; use starlark::codemap::FileSpan; +use starlark::environment::Globals; +use starlark::environment::Module; use starlark::syntax::AstModule; +use starlark::syntax::Dialect; use crate::interpreter::cycles::LoadCycleDescriptor; use crate::interpreter::dice_calculation_delegate::keys::EvalImportKey; @@ -62,6 +67,8 @@ enum DiceCalculationDelegateError { EvalBuildFileError(BuildFilePath), #[error("Error evaluating module: `{0}`")] EvalModuleError(String), + #[error("Error checking starlark stack size")] + CheckStarlarkStackSizeError, } #[async_trait] @@ -458,11 +465,81 @@ impl<'c> DiceCalculationDelegate<'c> { .await } + // In order to prevent non deterministic crashes + // we intentionally set off a starlark stack overflow, to make + // sure that starlark catches the overflow and reports an error + // before the native stack overflows + async fn check_starlark_stack_size(&self) -> anyhow::Result<()> { + #[derive(Debug, Display, Clone, Allocative, Eq, PartialEq, Hash)] + struct StarlarkStackSizeChecker; + + #[async_trait] + impl Key for StarlarkStackSizeChecker { + type Value = buck2_error::Result<()>; + + async fn compute( + &self, + ctx: &mut DiceComputations, + _cancellation: &CancellationContext, + ) -> Self::Value { + with_starlark_eval_provider( + ctx, + &mut StarlarkProfilerOrInstrumentation::disabled(), + "Check starlark stack size".to_owned(), + move |provider, _| { + let env = Module::new(); + let mut eval = provider.make(&env)?; + let content = indoc!( + r#" + def f(): + f() + f() + "# + ); + let ast = + AstModule::parse("x.star", content.to_owned(), &Dialect::Extended) + .map_err(BuckStarlarkError::new)?; + match eval.eval_module(ast, &Globals::standard()) { + Err(e) if e.to_string().contains("Starlark call stack overflow") => { + Ok(()) + } + Err(p) => Err(BuckStarlarkError::new(p).into()), + Ok(_) => { + Err(DiceCalculationDelegateError::CheckStarlarkStackSizeError + .into()) + } + } + }, + ) + .await?; + Ok(()) + } + + fn equality(x: &Self::Value, y: &Self::Value) -> bool { + match (x, y) { + (Ok(x), Ok(y)) => x == y, + _ => false, + } + } + + fn validity(x: &Self::Value) -> bool { + x.is_ok() + } + } + + self.ctx + .compute(&StarlarkStackSizeChecker) + .await? + .map_err(anyhow::Error::from) + } + pub async fn eval_build_file( &self, package: PackageLabel, profiler_instrumentation: &mut StarlarkProfilerOrInstrumentation<'_>, ) -> buck2_error::Result> { + self.check_starlark_stack_size().await?; + let listing = self.resolve_package_listing(package.dupe()).await?; let build_file_path = BuildFilePath::new(package.dupe(), listing.buildfile().to_owned()); diff --git a/app/buck2_interpreter_for_build/src/interpreter/extra_value.rs b/app/buck2_interpreter_for_build/src/interpreter/extra_value.rs new file mode 100644 index 0000000000000..34891436a7a06 --- /dev/null +++ b/app/buck2_interpreter_for_build/src/interpreter/extra_value.rs @@ -0,0 +1,108 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::cell::OnceCell; + +use allocative::Allocative; +use starlark::any::ProvidesStaticType; +use starlark::environment::FrozenModule; +use starlark::environment::Module; +use starlark::values::starlark_value; +use starlark::values::Freeze; +use starlark::values::Freezer; +use starlark::values::NoSerialize; +use starlark::values::OwnedFrozenValueTyped; +use starlark::values::StarlarkValue; +use starlark::values::Trace; +use starlark::values::ValueLike; + +use crate::interpreter::package_file_extra::FrozenPackageFileExtra; +use crate::interpreter::package_file_extra::PackageFileExtra; + +#[derive(buck2_error::Error, Debug)] +enum ExtraValueError { + #[error("Extra value is missing (internal error)")] + Missing, + #[error("Extra value had wrong type (internal error)")] + WrongType, +} + +/// `Module.extra_value` when evaluating build, bzl, package, and bxl files. +#[derive( + Default, + Debug, + NoSerialize, + derive_more::Display, + ProvidesStaticType, + Allocative, + Trace +)] +#[display(fmt = "{:?}", "self")] +pub(crate) struct ExtraValue<'v> { + /// Set when evaluating `PACKAGE` files. + pub(crate) package_extra: OnceCell>, +} + +#[derive( + Debug, + NoSerialize, + derive_more::Display, + ProvidesStaticType, + Allocative +)] +#[display(fmt = "{:?}", "self")] +pub(crate) struct FrozenExtraValue { + pub(crate) package_extra: Option, +} + +// TODO(nga): this does not need to be fully starlark_value, +// but we don't have lighter machinery for that. +#[starlark_value(type = "ExtraValue")] +impl<'v> StarlarkValue<'v> for ExtraValue<'v> {} + +#[starlark_value(type = "ExtraValue")] +impl<'v> StarlarkValue<'v> for FrozenExtraValue { + type Canonical = FrozenExtraValue; +} + +impl<'v> Freeze for ExtraValue<'v> { + type Frozen = FrozenExtraValue; + + fn freeze(self, freezer: &Freezer) -> anyhow::Result { + Ok(FrozenExtraValue { + package_extra: self + .package_extra + .into_inner() + .map(|p| p.freeze(freezer)) + .transpose()?, + }) + } +} + +impl<'v> ExtraValue<'v> { + pub(crate) fn get(module: &'v Module) -> anyhow::Result<&'v ExtraValue<'v>> { + Ok(module + .extra_value() + .ok_or(ExtraValueError::Missing)? + .downcast_ref() + .ok_or(ExtraValueError::WrongType)?) + } +} + +impl FrozenExtraValue { + pub(crate) fn get( + module: &FrozenModule, + ) -> anyhow::Result> { + Ok(module + .owned_extra_value() + .ok_or(ExtraValueError::Missing)? + .downcast() + .map_err(|_| ExtraValueError::WrongType)?) + } +} diff --git a/app/buck2_interpreter_for_build/src/interpreter/interpreter_for_cell.rs b/app/buck2_interpreter_for_build/src/interpreter/interpreter_for_cell.rs index 17635c884735b..0607073e1058a 100644 --- a/app/buck2_interpreter_for_build/src/interpreter/interpreter_for_cell.rs +++ b/app/buck2_interpreter_for_build/src/interpreter/interpreter_for_cell.rs @@ -23,8 +23,8 @@ use buck2_core::bzl::ImportPath; use buck2_core::cells::build_file_cell::BuildFileCell; use buck2_core::cells::cell_path::CellPath; use buck2_core::cells::CellAliasResolver; +use buck2_core::soft_error; use buck2_event_observer::humanized::HumanizedBytes; -use buck2_events::dispatch::console_warning; use buck2_events::dispatch::get_dispatcher; use buck2_interpreter::error::BuckStarlarkError; use buck2_interpreter::factory::StarlarkEvaluatorProvider; @@ -52,12 +52,13 @@ use starlark::codemap::FileSpan; use starlark::environment::FrozenModule; use starlark::environment::Module; use starlark::syntax::AstModule; -use starlark::values::OwnedFrozenValueTyped; +use starlark::values::OwnedFrozenRef; use crate::interpreter::build_context::BuildContext; use crate::interpreter::build_context::PerFileTypeContext; use crate::interpreter::bzl_eval_ctx::BzlEvalCtx; use crate::interpreter::cell_info::InterpreterCellInfo; +use crate::interpreter::extra_value::ExtraValue; use crate::interpreter::global_interpreter_state::GlobalInterpreterState; use crate::interpreter::module_internals::ModuleInternals; use crate::interpreter::package_file_extra::FrozenPackageFileExtra; @@ -70,10 +71,18 @@ struct StarlarkTabsError(OwnedStarlarkPath); #[derive(Debug, buck2_error::Error)] enum StarlarkPeakMemoryError { #[error( - "Starlark peak memory usage for {0} is `{1}` which exceeds the limit `{2}`! Please reduce memory usage to prevent OOMs. See https://fburl.com/starlark_peak_mem_warning for debugging tips." + "Starlark peak memory usage for {0} is {1} which exceeds the limit {2}! Please reduce memory usage to prevent OOMs. See {3} for debugging tips." )] #[buck2(user)] - ExceedsThreshold(BuildFilePath, HumanizedBytes, HumanizedBytes), + ExceedsThreshold(BuildFilePath, HumanizedBytes, HumanizedBytes, String), +} + +#[derive(Debug, buck2_error::Error)] +enum StarlarkPeakMemorySoftError { + #[error( + "Starlark peak memory usage for {0} is {1} which is over 50% of the limit {2}! Consider investigating what takes too much memory: {3}." + )] + CloseToThreshold(BuildFilePath, HumanizedBytes, HumanizedBytes, String), } #[derive(Debug, buck2_error::Error)] @@ -121,6 +130,13 @@ impl ParseData { } } +pub fn get_starlark_warning_link() -> &'static str { + if buck2_core::is_open_source() { + "https://buck2.build/docs/users/faq/starlark_peak_mem" + } else { + "https://fburl.com/starlark_peak_mem_warning" + } +} /// Interpreter for build files. /// /// The Interpreter is responsible for parsing files to an AST and then @@ -295,6 +311,8 @@ impl InterpreterForCell { } } + env.set_extra_value(env.heap().alloc_complex(ExtraValue::default())); + Ok(env) } @@ -580,10 +598,10 @@ impl InterpreterForCell { )? .additional; - let extra: Option> = - if env.extra_value().is_some() { - // Only freeze if there's extra, otherwise we will needlessly freeze globals. - // TODO(nga): add API to only freeze extra. + let extra: Option> = + if ExtraValue::get(&env)?.package_extra.get().is_some() { + // Only freeze if there's something to freeze, otherwise we will needlessly freeze + // globals. TODO(nga): add API to only freeze extra. let env = env.freeze()?; FrozenPackageFileExtra::get(&env)? } else { @@ -631,10 +649,9 @@ impl InterpreterForCell { let internals = build_ctx.additional.into_build()?; let starlark_peak_allocated_bytes = env.heap().peak_allocated_bytes() as u64; - // TODO(ezgi): err if we cannot parse as bool let starlark_peak_mem_check_enabled = root_buckconfig - .get("buck2", "check_starlark_peak_memory") - .map_or(false, |value| value.map_or(false, |v| &*v == "true")); + .parse("buck2", "check_starlark_peak_memory")? + .unwrap_or(false); let default_limit = 2 * (1 << 30); let starlark_mem_limit = build_ctx .starlark_peak_allocated_byte_limit @@ -646,20 +663,21 @@ impl InterpreterForCell { build_file.to_owned(), HumanizedBytes::fixed_width(starlark_peak_allocated_bytes), HumanizedBytes::fixed_width(starlark_mem_limit), + get_starlark_warning_link().to_owned(), ) .into()) } else if starlark_peak_mem_check_enabled && starlark_peak_allocated_bytes > starlark_mem_limit / 2 { - console_warning( - format!( - "Starlark peak memory usage for {} is `{}` which is over `50`% of the limit `{}`! Consider investigating what takes too much memory: https://fburl.com/starlark_peak_mem_warning.", - build_file, + soft_error!( + "starlark_memory_usage_over_soft_limit", + StarlarkPeakMemorySoftError::CloseToThreshold( + build_file.clone(), HumanizedBytes::fixed_width(starlark_peak_allocated_bytes), - HumanizedBytes::fixed_width(starlark_mem_limit) - ) - .to_owned(), - ); + HumanizedBytes::fixed_width(starlark_mem_limit), + get_starlark_warning_link().to_owned() + ).into(), quiet: true + )?; Ok(EvaluationResultWithStats { result: EvaluationResult::from(internals), diff --git a/app/buck2_interpreter_for_build/src/interpreter/mod.rs b/app/buck2_interpreter_for_build/src/interpreter/mod.rs index 826568a99a928..066b628790032 100644 --- a/app/buck2_interpreter_for_build/src/interpreter/mod.rs +++ b/app/buck2_interpreter_for_build/src/interpreter/mod.rs @@ -17,6 +17,7 @@ pub mod configuror; pub mod context; pub mod cycles; pub mod dice_calculation_delegate; +mod extra_value; pub mod functions; pub mod global_interpreter_state; pub mod globals; diff --git a/app/buck2_interpreter_for_build/src/interpreter/package_file_extra.rs b/app/buck2_interpreter_for_build/src/interpreter/package_file_extra.rs index 851b6360d277c..389125c2e0e5a 100644 --- a/app/buck2_interpreter_for_build/src/interpreter/package_file_extra.rs +++ b/app/buck2_interpreter_for_build/src/interpreter/package_file_extra.rs @@ -24,24 +24,19 @@ use starlark::values::Freeze; use starlark::values::Freezer; use starlark::values::FrozenValue; use starlark::values::NoSerialize; +use starlark::values::OwnedFrozenRef; use starlark::values::OwnedFrozenValue; -use starlark::values::OwnedFrozenValueTyped; use starlark::values::StarlarkValue; use starlark::values::Trace; use starlark::values::Tracer; use starlark::values::Value; -use starlark::values::ValueLike; use starlark_map::small_map::SmallMap; +use crate::interpreter::extra_value::ExtraValue; +use crate::interpreter::extra_value::FrozenExtraValue; use crate::super_package::package_value::FrozenStarlarkPackageValue; use crate::super_package::package_value::StarlarkPackageValue; -#[derive(Debug, buck2_error::Error)] -enum PackageFileExtraError { - #[error("Wrong type of frozen package extra (internal error)")] - WrongTypeOfFrozenExtra, -} - /// `Module.extra_value` when evaluating `PACKAGE` file. #[derive( Default, @@ -128,36 +123,18 @@ impl<'v> Freeze for PackageFileExtra<'v> { impl<'v> PackageFileExtra<'v> { pub fn get_or_init(eval: &mut Evaluator<'v, '_>) -> anyhow::Result<&'v PackageFileExtra<'v>> { - match eval.module().extra_value() { - None => { - let extra = eval.heap().alloc_complex(PackageFileExtra::default()); - eval.module().set_extra_value(extra); - let extra = extra - .downcast_ref_err::() - .context("(internal error)")?; - Ok(extra) - } - Some(extra) => { - let extra = extra - .downcast_ref_err::() - .context("(internal error)")?; - Ok(extra) - } - } + Ok(ExtraValue::get(eval.module())? + .package_extra + .get_or_init(Default::default)) } } impl FrozenPackageFileExtra { pub(crate) fn get( module: &FrozenModule, - ) -> anyhow::Result>> { - match module.owned_extra_value() { - None => Ok(None), - Some(extra) => { - Ok(Some(extra.downcast().map_err(|_| { - PackageFileExtraError::WrongTypeOfFrozenExtra - })?)) - } - } + ) -> anyhow::Result>> { + Ok(FrozenExtraValue::get(module)? + .into_owned_frozen_ref() + .try_map_option(|x| x.package_extra.as_ref())) } } diff --git a/app/buck2_interpreter_for_build/src/interpreter/selector.rs b/app/buck2_interpreter_for_build/src/interpreter/selector.rs index b38376575c0f1..bc024e2437cc1 100644 --- a/app/buck2_interpreter_for_build/src/interpreter/selector.rs +++ b/app/buck2_interpreter_for_build/src/interpreter/selector.rs @@ -246,7 +246,7 @@ pub fn register_select(globals: &mut GlobalsBuilder) { /// def increment_items(a): /// return [v + 1 for v in a] /// - /// select_map([1, 2] + select({"c": [2]}}, increment_items) == [2, 3] + select({"c": [3]}) + /// select_map([1, 2] + select({"c": [2]}), increment_items) == [2, 3] + select({"c": [3]}) /// ``` fn select_map<'v>( #[starlark(require = pos)] d: Value<'v>, diff --git a/app/buck2_interpreter_for_build/src/nodes/check_within_view.rs b/app/buck2_interpreter_for_build/src/nodes/check_within_view.rs index 0194d229b8b72..ac5df78df02a0 100644 --- a/app/buck2_interpreter_for_build/src/nodes/check_within_view.rs +++ b/app/buck2_interpreter_for_build/src/nodes/check_within_view.rs @@ -43,6 +43,7 @@ enum CheckWithinViewError { _0, indented_within_view(_1) )] + #[buck2(tag = Visibility)] DepNotWithinView(TargetLabel, WithinViewSpecification), } diff --git a/app/buck2_interpreter_for_build/src/super_package/eval_ctx.rs b/app/buck2_interpreter_for_build/src/super_package/eval_ctx.rs index 76bc52251439a..f7336faf2d4cf 100644 --- a/app/buck2_interpreter_for_build/src/super_package/eval_ctx.rs +++ b/app/buck2_interpreter_for_build/src/super_package/eval_ctx.rs @@ -16,8 +16,8 @@ use buck2_node::super_package::SuperPackage; use buck2_node::visibility::VisibilitySpecification; use buck2_node::visibility::WithinViewSpecification; use dupe::Dupe; +use starlark::values::OwnedFrozenRef; use starlark::values::OwnedFrozenValue; -use starlark::values::OwnedFrozenValueTyped; use starlark_map::small_map::SmallMap; use crate::interpreter::package_file_extra::FrozenPackageFileExtra; @@ -43,7 +43,7 @@ pub struct PackageFileEvalCtx { impl PackageFileEvalCtx { fn cfg_constructor( - extra: Option<&OwnedFrozenValueTyped>, + extra: Option<&OwnedFrozenRef>, ) -> anyhow::Result>> { let Some(extra) = extra else { return Ok(None); @@ -61,7 +61,7 @@ impl PackageFileEvalCtx { pub(crate) fn build_super_package( self, - extra: Option>, + extra: Option>, ) -> anyhow::Result { let cfg_constructor = Self::cfg_constructor(extra.as_ref())?; diff --git a/app/buck2_miniperf_proto/src/lib.rs b/app/buck2_miniperf_proto/src/lib.rs index 9922b28677eee..a293b698e7e8c 100644 --- a/app/buck2_miniperf_proto/src/lib.rs +++ b/app/buck2_miniperf_proto/src/lib.rs @@ -79,7 +79,7 @@ impl MiniperfCounter { } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_node/BUCK b/app/buck2_node/BUCK index 14c3ca2f10475..dc178393bcf47 100644 --- a/app/buck2_node/BUCK +++ b/app/buck2_node/BUCK @@ -13,7 +13,6 @@ rust_library( deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:async-trait", - "fbsource//third-party/rust:derivative", "fbsource//third-party/rust:derive_more", "fbsource//third-party/rust:either", "fbsource//third-party/rust:fnv", @@ -25,9 +24,11 @@ rust_library( "fbsource//third-party/rust:serde_json", "fbsource//third-party/rust:static_assertions", "fbsource//third-party/rust:strsim", + "fbsource//third-party/rust:triomphe", "//buck2/allocative/allocative:allocative", "//buck2/app/buck2_common:buck2_common", "//buck2/app/buck2_core:buck2_core", + "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_error:buck2_error", "//buck2/app/buck2_events:buck2_events", "//buck2/app/buck2_query:buck2_query", diff --git a/app/buck2_node/Cargo.toml b/app/buck2_node/Cargo.toml index abe01b92cb27e..863652991c911 100644 --- a/app/buck2_node/Cargo.toml +++ b/app/buck2_node/Cargo.toml @@ -9,7 +9,6 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -derivative = { workspace = true } derive_more = { workspace = true } either = { workspace = true } fnv = { workspace = true } @@ -21,6 +20,7 @@ serde = { workspace = true } serde_json = { workspace = true } static_assertions = { workspace = true } strsim = { workspace = true } +triomphe = { workspace = true } allocative = { workspace = true } cmp_any = { workspace = true } @@ -32,6 +32,7 @@ starlark_map = { workspace = true } buck2_common = { workspace = true } buck2_core = { workspace = true } +buck2_data = { workspace = true } buck2_error = { workspace = true } buck2_events = { workspace = true } buck2_query = { workspace = true } diff --git a/app/buck2_node/src/cfg_constructor.rs b/app/buck2_node/src/cfg_constructor.rs index 132a9f5170d78..af8538adb0250 100644 --- a/app/buck2_node/src/cfg_constructor.rs +++ b/app/buck2_node/src/cfg_constructor.rs @@ -20,7 +20,8 @@ use dice::DiceComputations; use crate::metadata::key::MetadataKeyRef; use crate::metadata::value::MetadataValue; -use crate::nodes::unconfigured::TargetNode; +use crate::nodes::unconfigured::TargetNodeRef; +use crate::rule_type::RuleType; use crate::super_package::SuperPackage; /// Trait for configuration constructor functions. @@ -34,6 +35,7 @@ pub trait CfgConstructorImpl: Send + Sync + Debug + Allocative { package_cfg_modifiers: Option<&'a MetadataValue>, target_cfg_modifiers: Option<&'a MetadataValue>, cli_modifiers: &'a [String], + rule_type: &'a RuleType, ) -> Pin> + Send + 'a>>; /// Returns the metadata key used to encode modifiers in PACKAGE values and metadata attribute @@ -57,8 +59,10 @@ pub trait CfgConstructorCalculationImpl: Send + Sync + 'static { async fn eval_cfg_constructor( &self, ctx: &DiceComputations, - target: &TargetNode, + target: TargetNodeRef<'_>, super_package: &SuperPackage, cfg: ConfigurationData, + cli_modifiers: &Arc>, + rule_name: &RuleType, ) -> anyhow::Result; } diff --git a/app/buck2_node/src/configuration/toolchain_constraints.rs b/app/buck2_node/src/configuration/toolchain_constraints.rs index 142614a9983a1..0ebe286097b54 100644 --- a/app/buck2_node/src/configuration/toolchain_constraints.rs +++ b/app/buck2_node/src/configuration/toolchain_constraints.rs @@ -10,35 +10,48 @@ use std::sync::Arc; use allocative::Allocative; -use buck2_core::configuration::compatibility::IncompatiblePlatformReason; -use buck2_core::execution_types::execution::ExecutionPlatform; +use buck2_core::target::label::TargetLabel; use dupe::Dupe; -use starlark_map::small_map::SmallMap; +use dupe::IterDupedExt; /// The constraint introduced on execution platform resolution by /// a toolchain rule (reached via a toolchain_dep). -#[derive(Dupe, Clone, PartialEq, Eq, Allocative)] -pub struct ToolchainConstraints { - // We know the set of execution platforms is fixed throughout the build, - // so we can record just the ones we are incompatible with, - // and assume all others we _are_ compatible with. - incompatible: Arc>>, +#[derive(Debug, Dupe, Clone, PartialEq, Eq, Hash, Allocative)] +pub struct ToolchainConstraints(Arc); + +#[derive(Debug, PartialEq, Eq, Hash, Allocative)] +struct ToolchainConstraintsImpl { + exec_deps: Vec, + exec_compatible_with: Vec, } impl ToolchainConstraints { - pub fn new(incompatible: SmallMap>) -> Self { - Self { - incompatible: Arc::new(incompatible), - } + pub fn new( + exec_deps: &[TargetLabel], + exec_compatible_with: &[TargetLabel], + inherited_toolchains: &[ToolchainConstraints], + ) -> Self { + Self(Arc::new(ToolchainConstraintsImpl { + exec_deps: inherited_toolchains + .iter() + .flat_map(|i| &i.0.exec_deps) + .chain(exec_deps) + .duped() + .collect(), + exec_compatible_with: inherited_toolchains + .iter() + .flat_map(|i| &i.0.exec_compatible_with) + .chain(exec_compatible_with) + .duped() + .collect(), + })) + } + + pub fn exec_deps(&self) -> impl Iterator { + self.0.exec_deps.iter() } - pub fn allows( - &self, - exec_platform: &ExecutionPlatform, - ) -> Result<(), Arc> { - match self.incompatible.get(exec_platform) { - None => Ok(()), - Some(e) => Err(e.dupe()), - } + pub fn exec_compatible_with(&self) -> impl Iterator { + self.0.exec_compatible_with.iter() } } diff --git a/app/buck2_node/src/configured_universe.rs b/app/buck2_node/src/configured_universe.rs index 97ae814e77a98..8229bd463750b 100644 --- a/app/buck2_node/src/configured_universe.rs +++ b/app/buck2_node/src/configured_universe.rs @@ -19,51 +19,61 @@ use buck2_core::pattern::pattern_type::TargetPatternExtra; use buck2_core::pattern::PackageSpec; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::label::TargetLabel; -use buck2_core::target::name::TargetName; +use buck2_core::target::name::TargetNameRef; +use buck2_events::dispatch::span; use buck2_query::query::syntax::simple::eval::label_indexed::LabelIndexed; use buck2_query::query::syntax::simple::eval::set::TargetSet; -use derivative::Derivative; +use buck2_util::self_ref::RefData; +use buck2_util::self_ref::SelfRef; use dupe::Dupe; -use dupe::IterDupedExt; use either::Either; use itertools::Itertools; use crate::nodes::configured::ConfiguredTargetNode; +use crate::nodes::configured::ConfiguredTargetNodeRef; use crate::nodes::configured_node_visit_all_deps::configured_node_visit_all_deps; +#[derive(Debug)] +struct CqueryUniverseInner<'a> { + targets: BTreeMap< + PackageLabel, + BTreeMap<&'a TargetNameRef, BTreeSet>>>, + >, +} + +struct CqueryUniverseInnerType; + +impl RefData for CqueryUniverseInnerType { + type Data<'a> = CqueryUniverseInner<'a>; +} + /// Subset of targets `cquery` command works with. /// /// Targets are resolved in the universe, and file owners are also resolved in the universe. -#[derive(Allocative, Derivative, Clone)] -#[derivative(Debug)] +#[derive(Allocative, Debug)] pub struct CqueryUniverse { - targets: - BTreeMap>>>, + data: SelfRef, } -impl CqueryUniverse { +impl<'a> CqueryUniverseInner<'a> { pub fn new( targets: BTreeMap< PackageLabel, - BTreeMap>>, + BTreeMap<&'a TargetNameRef, BTreeSet>>>, >, - ) -> CqueryUniverse { - CqueryUniverse { targets } - } - - pub fn len(&self) -> usize { - self.targets.values().map(|e| e.values().len()).sum() + ) -> Self { + CqueryUniverseInner { targets } } - pub async fn build( - universe: &TargetSet, - ) -> anyhow::Result { + fn build_inner( + universe: &'a TargetSet, + ) -> anyhow::Result> { let mut targets: BTreeMap< PackageLabel, - BTreeMap>>, + BTreeMap<&TargetNameRef, BTreeSet>>, > = BTreeMap::new(); - configured_node_visit_all_deps(universe.iter().duped(), |target| { + configured_node_visit_all_deps(universe.iter().map(|t| t.as_ref()), |target| { let label = target.label(); let package_targets: &mut _ = targets .entry(label.pkg().dupe()) @@ -72,17 +82,36 @@ impl CqueryUniverse { let nodes: &mut _ = match package_targets.get_mut(label.name()) { Some(v) => v, None => package_targets - .entry(label.name().to_owned()) + .entry(label.name()) .or_insert_with(BTreeSet::new), }; - nodes.insert(LabelIndexed(target)); + let inserted = nodes.insert(LabelIndexed(target)); + assert!(inserted, "Visited targets must be unique"); + }); - Ok(()) - }) - .await?; + Ok(CqueryUniverseInner::new(targets)) + } +} - Ok(CqueryUniverse::new(targets)) +impl CqueryUniverse { + pub fn len(&self) -> usize { + self.data + .data() + .targets + .values() + .map(|e| e.values().len()) + .sum() + } + + pub fn build(universe: &TargetSet) -> anyhow::Result { + span(buck2_data::CqueryUniverseBuildStart {}, || { + let r = SelfRef::try_new(universe.clone(), |universe| { + CqueryUniverseInner::build_inner(universe) + }) + .map(|data| CqueryUniverse { data }); + (r, buck2_data::CqueryUniverseBuildEnd {}) + }) } pub fn get( @@ -93,7 +122,7 @@ impl CqueryUniverse { for (package, spec) in &resolved_pattern.specs { targets.extend( self.get_from_package(package.dupe(), spec) - .map(|(node, TargetPatternExtra)| node), + .map(|(node, TargetPatternExtra)| node.to_owned()), ); } targets @@ -110,11 +139,13 @@ impl CqueryUniverse { let package = label.pkg(); let name = label.name(); let results = self + .data + .data() .targets .get(&package) .into_iter() .flat_map(move |package_universe| package_universe.get(name).into_iter().flatten()) - .map(|node| node.0.clone()); + .map(|node| node.0.to_owned()); configured_nodes.extend(results); } @@ -141,29 +172,34 @@ impl CqueryUniverse { &'a self, package: PackageLabel, spec: &'a PackageSpec

, - ) -> impl Iterator + 'a { - self.targets + ) -> impl Iterator, P)> + 'a { + self.data + .data() + .targets .get(&package) .into_iter() .flat_map(move |package_universe| match spec { PackageSpec::Targets(names) => { Either::Left(names.iter().flat_map(|(name, extra)| { - package_universe.get(name).into_iter().flat_map(|nodes| { - nodes.iter().filter_map(|node| { - if extra.matches_cfg(node.0.label().cfg()) { - Some((&node.0, extra.clone())) - } else { - None - } + package_universe + .get(name.as_ref()) + .into_iter() + .flat_map(|nodes| { + nodes.iter().filter_map(|node| { + if extra.matches_cfg(node.0.label().cfg()) { + Some((node.0, extra.clone())) + } else { + None + } + }) }) - }) })) } PackageSpec::All => Either::Right( package_universe .values() .flatten() - .map(|node| (&node.0, P::default())), + .map(|node| (node.0, P::default())), ), }) } @@ -182,13 +218,13 @@ impl CqueryUniverse { // We do it because the map is by `Package`, // and `BTreeMap` does not allow lookup by equivalent key. let package = PackageLabel::from_cell_path(package); - let package_data = match self.targets.get(&package) { + let package_data = match self.data.data().targets.get(&package) { None => continue, Some(package_data) => package_data, }; for node in package_data.values().flatten() { if node.0.inputs().contains(path) { - nodes.push(node.0.dupe()); + nodes.push(node.0.to_owned()); } } } @@ -251,7 +287,6 @@ mod tests { target_label.dupe(), "idris_library", )])) - .await .unwrap(); let provider_label = ConfiguredProvidersLabel::new(target_label, providers_name()); diff --git a/app/buck2_node/src/load_patterns.rs b/app/buck2_node/src/load_patterns.rs index 4b49fc0c719e8..41caf74e6357a 100644 --- a/app/buck2_node/src/load_patterns.rs +++ b/app/buck2_node/src/load_patterns.rs @@ -33,6 +33,7 @@ use itertools::Itertools; use crate::nodes::eval_result::EvaluationResult; use crate::nodes::frontend::TargetGraphCalculation; use crate::nodes::unconfigured::TargetNode; +use crate::nodes::unconfigured::TargetNodeRef; use crate::super_package::SuperPackage; #[derive(Debug, buck2_error::Error)] @@ -120,16 +121,16 @@ pub struct PackageLoadedPatterns { } impl PackageLoadedPatterns { - pub fn iter(&self) -> impl Iterator { - self.targets.iter() + pub fn iter(&self) -> impl Iterator)> { + self.targets.iter().map(|(k, v)| (k, v.as_ref())) } pub fn keys(&self) -> impl Iterator { self.targets.keys() } - pub fn values(&self) -> impl Iterator { - self.targets.values() + pub fn values(&self) -> impl Iterator> { + self.targets.values().map(|v| v.as_ref()) } pub fn into_values(self) -> impl Iterator { @@ -165,11 +166,13 @@ impl LoadedPatterns { self.results.into_iter() } - pub fn iter_loaded_targets(&self) -> impl Iterator> { + pub fn iter_loaded_targets( + &self, + ) -> impl Iterator>> { self.results .values() .map(|result| match result { - Ok(pkg) => Ok(pkg.targets.values()), + Ok(pkg) => Ok(pkg.targets.values().map(|t| t.as_ref())), Err(e) => Err(e.dupe()), }) .flatten_ok() diff --git a/app/buck2_node/src/nodes/configured.rs b/app/buck2_node/src/nodes/configured.rs index e358d284dd0d6..fab334a71a7cb 100644 --- a/app/buck2_node/src/nodes/configured.rs +++ b/app/buck2_node/src/nodes/configured.rs @@ -77,8 +77,10 @@ use crate::rule_type::StarlarkRuleType; /// in the node, instead the node just stores the base TargetNode and a configuration for /// resolving the attributes. This saves memory, but users should try avoid repeatedly /// requesting the same information. -#[derive(Debug, Clone, Dupe, Eq, PartialEq, Hash, Allocative)] -pub struct ConfiguredTargetNode(Arc>); +#[derive(Debug, Clone, Eq, PartialEq, Hash, Allocative)] +pub struct ConfiguredTargetNode(triomphe::Arc>); + +impl Dupe for ConfiguredTargetNode {} #[derive(Debug, Eq, PartialEq, Hash, Allocative)] enum TargetNodeOrForward { @@ -170,7 +172,7 @@ impl TargetNodeOrForward { // 3. deps could probably be approximated a diff against the targetnode's deps #[derive(Eq, PartialEq, Hash, Allocative)] struct ConfiguredTargetNodeData { - label: ConfiguredTargetLabel, + label: Hashed, target_node: TargetNodeOrForward, resolved_configuration: ResolvedConfiguration, resolved_transition_configurations: OrderedMap, Arc>, @@ -178,8 +180,7 @@ struct ConfiguredTargetNodeData { // Deps includes regular deps and transitioned deps, // but excludes exec deps or configuration deps. // TODO(cjhopman): Should this be a diff against the node's deps? - deps: ConfiguredTargetNodeDeps, - exec_deps: ConfiguredTargetNodeDeps, + all_deps: ConfiguredTargetNodeDeps, platform_cfgs: OrderedMap, // TODO(JakobDegen): Consider saving some memory by using a more tset like representation of // the plugin lists @@ -232,14 +233,13 @@ impl ConfiguredTargetNode { platform_cfgs: OrderedMap, plugin_lists: PluginLists, ) -> Self { - Self(Arc::new(Hashed::new(ConfiguredTargetNodeData { - label: name, + Self(triomphe::Arc::new(Hashed::new(ConfiguredTargetNodeData { + label: Hashed::new(name), target_node: TargetNodeOrForward::TargetNode(target_node), resolved_configuration, resolved_transition_configurations: resolved_tr_configurations, execution_platform_resolution, - deps: ConfiguredTargetNodeDeps(deps.into_boxed_slice()), - exec_deps: ConfiguredTargetNodeDeps(exec_deps.into_boxed_slice()), + all_deps: ConfiguredTargetNodeDeps::new(deps, exec_deps), platform_cfgs, plugin_lists, }))) @@ -269,9 +269,9 @@ impl ConfiguredTargetNode { let configured_providers_label = providers_label.configure(transitioned_node.label().cfg().dupe()); - Ok(ConfiguredTargetNode(Arc::new(Hashed::new( + Ok(ConfiguredTargetNode(triomphe::Arc::new(Hashed::new( ConfiguredTargetNodeData { - label: name.dupe(), + label: Hashed::new(name.dupe()), target_node: TargetNodeOrForward::Forward( CoercedAttr::ConfiguredDep(Box::new(DepAttr { attr_type: DepAttrType::new( @@ -295,8 +295,7 @@ impl ConfiguredTargetNode { .execution_platform_resolution() .dupe(), plugin_lists: transitioned_node.plugin_lists().clone(), - deps: ConfiguredTargetNodeDeps(Box::new([transitioned_node])), - exec_deps: ConfiguredTargetNodeDeps(Box::new([])), + all_deps: ConfiguredTargetNodeDeps::new(vec![transitioned_node], vec![]), platform_cfgs: OrderedMap::new(), }, )))) @@ -340,84 +339,36 @@ impl ConfiguredTargetNode { /// later in the build process). // TODO(cjhopman): Should this include configuration deps? Should it include the configuration deps that were inspected resolving selects? pub fn deps(&self) -> impl Iterator { - self.0.deps.iter().chain(self.0.exec_deps.iter()) + self.0.all_deps.all_deps.iter() } pub fn toolchain_deps(&self) -> impl Iterator { // Since we validate that all toolchain dependencies are of kind Toolchain, // we can use that to filter the deps. self.0 - .deps + .all_deps + .deps() .iter() .filter(|x| x.rule_kind() == RuleKind::Toolchain) } pub fn inputs(&self) -> impl Iterator + '_ { - struct InputsCollector { - inputs: Vec, - } - impl ConfiguredAttrTraversal for InputsCollector { - fn dep(&mut self, _dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { - Ok(()) - } - - fn input(&mut self, path: BuckPathRef) -> anyhow::Result<()> { - self.inputs.push(path.to_cell_path()); - Ok(()) - } - } - let mut traversal = InputsCollector { inputs: Vec::new() }; - for a in self.attrs(AttrInspectOptions::All) { - a.traverse(self.label().pkg(), &mut traversal) - .expect("inputs collector shouldn't return errors"); - } - traversal.inputs.into_iter() + self.as_ref().inputs() } - // TODO(cjhopman): switch to for_each_query? + #[inline] pub fn queries( &self, - ) -> impl Iterator)> { - struct Traversal { - queries: Vec<(String, ResolvedQueryLiterals)>, - } - let mut traversal = Traversal { - queries: Vec::new(), - }; - impl ConfiguredAttrTraversal for Traversal { - fn dep(&mut self, _dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { - // ignored. - Ok(()) - } - - fn query( - &mut self, - query: &str, - resolved_literals: &ResolvedQueryLiterals, - ) -> anyhow::Result<()> { - self.queries - .push((query.to_owned(), resolved_literals.clone())); - Ok(()) - } - } - - for a in self.attrs(AttrInspectOptions::All) { - // Optimization. - if !a.attr.coercer().0.may_have_queries { - continue; - } - - a.traverse(self.label().pkg(), &mut traversal).unwrap(); - } - traversal.queries.into_iter() + ) -> impl Iterator)> + '_ { + self.as_ref().queries() } pub fn target_deps(&self) -> impl Iterator { - self.0.deps.iter() + self.0.all_deps.deps().iter() } pub fn exec_deps(&self) -> impl Iterator { - self.0.exec_deps.iter() + self.0.all_deps.exec_deps().iter() } /// Return the `tests` declared for this target configured in same target platform as this target. @@ -450,6 +401,11 @@ impl ConfiguredTargetNode { &self.0.label } + #[inline] + pub fn hashed_label(&self) -> Hashed<&ConfiguredTargetLabel> { + self.0.label.as_ref() + } + pub fn rule_type(&self) -> &RuleType { self.0.target_node.rule_type() } @@ -466,7 +422,203 @@ impl ConfiguredTargetNode { self.0.target_node.is_visible_to(target) } + #[inline] pub fn special_attrs(&self) -> impl Iterator { + self.as_ref().special_attrs() + } + + #[inline] + pub fn oncall(&self) -> Option<&str> { + self.as_ref().oncall() + } + + pub fn attrs<'a>( + &'a self, + opts: AttrInspectOptions, + ) -> impl Iterator> + 'a { + self.as_ref().attrs(opts) + } + + pub fn get<'a>( + &'a self, + attr: &str, + opts: AttrInspectOptions, + ) -> Option> { + self.as_ref().get(attr, opts) + } + + pub fn call_stack(&self) -> Option { + match &self.0.target_node { + TargetNodeOrForward::TargetNode(n) => n.call_stack(), + TargetNodeOrForward::Forward(_, n) => n.call_stack(), + } + } + + /// Hash the fields that impact how this target is built. + /// Don't do any recursive hashing of the dependencies. + /// Hashes the attributes _after_ configuration, so changing unconfigured branches that + /// are not taken will not change the hash. + pub fn target_hash(&self, state: &mut H) { + self.label().hash(state); + self.rule_type().hash(state); + self.attrs(AttrInspectOptions::All).for_each(|x| { + // We deliberately don't hash the attribute, as if the value being passed to analysis + // stays the same, we don't care if the attribute that generated it changed. + x.name.hash(state); + x.value.hash(state); + }); + } + + /// If this node is a forward node, return the target it forwards to. + pub fn forward_target(&self) -> Option<&ConfiguredTargetNode> { + match &self.0.target_node { + TargetNodeOrForward::TargetNode(_) => None, + TargetNodeOrForward::Forward(_, n) => Some(n), + } + } + + #[inline] + pub fn uses_plugins(&self) -> &[PluginKind] { + self.as_ref().uses_plugins() + } + + pub fn plugin_lists(&self) -> &PluginLists { + &self.0.plugin_lists + } + + #[inline] + pub fn as_ref(&self) -> ConfiguredTargetNodeRef<'_> { + ConfiguredTargetNodeRef(triomphe::Arc::borrow_arc(&self.0)) + } + + pub fn ptr_eq(&self, other: &Self) -> bool { + triomphe::Arc::ptr_eq(&self.0, &other.0) + } +} + +/// The representation of the deps for a ConfiguredTargetNode. Provides the operations we require +/// (iteration, eq, and hash), but guarantees those aren't recursive of the dep nodes' data. +#[derive(Allocative)] +struct ConfiguredTargetNodeDeps { + deps_count: usize, + /// (target deps and toolchain deps) followed by `exec_deps`. + all_deps: Box<[ConfiguredTargetNode]>, +} + +impl ConfiguredTargetNodeDeps { + fn new(deps: Vec, exec_deps: Vec) -> Self { + if deps.is_empty() { + ConfiguredTargetNodeDeps { + deps_count: 0, + all_deps: exec_deps.into_boxed_slice(), + } + } else if exec_deps.is_empty() { + ConfiguredTargetNodeDeps { + deps_count: deps.len(), + all_deps: deps.into_boxed_slice(), + } + } else { + ConfiguredTargetNodeDeps { + deps_count: deps.len(), + all_deps: deps + .into_iter() + .chain(exec_deps) + .collect::>() + .into_boxed_slice(), + } + } + } + + fn deps(&self) -> &[ConfiguredTargetNode] { + &self.all_deps[..self.deps_count] + } + + fn exec_deps(&self) -> &[ConfiguredTargetNode] { + &self.all_deps[self.deps_count..] + } +} + +/// We only do Arc comparisons here. The rationale is that each ConfiguredTargetNode should only +/// ever exist in one instance in the graph, so if the ptr eq doesn't match, then we don't do a +/// deep comparison. +impl PartialEq for ConfiguredTargetNodeDeps { + fn eq(&self, other: &Self) -> bool { + let ConfiguredTargetNodeDeps { + deps_count, + all_deps, + } = self; + *deps_count == other.deps_count && all_deps.len() == other.all_deps.len() && { + let it1 = all_deps.iter(); + let it2 = other.all_deps.iter(); + it1.zip(it2).all(|(x, y)| triomphe::Arc::ptr_eq(&x.0, &y.0)) + } + } +} + +impl Eq for ConfiguredTargetNodeDeps {} + +/// This has historically only hashed the labels. This may or may not be right, because it means +/// two nodes that reference different instances of a given dependency will hash equal, even if +/// the dependency has definitely changed (e.g. because its own deps changed). +impl Hash for ConfiguredTargetNodeDeps { + fn hash(&self, state: &mut H) { + let ConfiguredTargetNodeDeps { + deps_count, + all_deps, + } = self; + state.write_usize(*deps_count); + for node in &**all_deps { + node.label().hash(state); + } + } +} + +/// Like `&ConfiguredTargetNode`, but cheaper (one fewer indirection). +#[derive(Debug, Copy, Clone)] +pub struct ConfiguredTargetNodeRef<'a>(triomphe::ArcBorrow<'a, Hashed>); + +impl<'a> Dupe for ConfiguredTargetNodeRef<'a> {} + +impl<'a> ConfiguredTargetNodeRef<'a> { + #[inline] + pub fn to_owned(self) -> ConfiguredTargetNode { + ConfiguredTargetNode(triomphe::ArcBorrow::clone_arc(&self.0)) + } + + #[inline] + pub fn deps(self) -> impl Iterator { + self.0.get().all_deps.all_deps.iter() + } + + #[inline] + pub fn ptr_eq(self, other: Self) -> bool { + triomphe::ArcBorrow::ptr_eq(&self.0, &other.0) + } + + #[inline] + pub fn label(self) -> &'a ConfiguredTargetLabel { + self.0.get().label.key() + } + + #[inline] + pub fn hashed_label(self) -> Hashed<&'a ConfiguredTargetLabel> { + self.0.get().label.as_ref() + } + + fn attr_configuration_context(self) -> AttrConfigurationContextImpl<'a> { + AttrConfigurationContextImpl::new( + &self.0.get().resolved_configuration, + self.0.get().execution_platform_resolution.cfg(), + &self.0.get().resolved_transition_configurations, + &self.0.get().platform_cfgs, + ) + } + + pub fn oncall(self) -> Option<&'a str> { + self.0.get().target_node.oncall() + } + + pub fn special_attrs(self) -> impl Iterator { let typ_attr = ConfiguredAttr::String(StringLiteral(self.rule_type().name().into())); let deps_attr = ConfiguredAttr::List( self.deps() @@ -510,82 +662,99 @@ impl ConfiguredTargetNode { .into_iter() } - pub fn oncall(&self) -> Option<&str> { - self.0.target_node.oncall() - } - - fn attr_configuration_context(&self) -> AttrConfigurationContextImpl { - AttrConfigurationContextImpl::new( - &self.0.resolved_configuration, - self.0.execution_platform_resolution.cfg(), - &self.0.resolved_transition_configurations, - &self.0.platform_cfgs, - ) - } - - pub fn attrs<'a>( - &'a self, + pub fn attrs( + self, opts: AttrInspectOptions, ) -> impl Iterator> + 'a { - self.0.target_node.attrs(opts).map(move |a| { + self.0.get().target_node.attrs(opts).map(move |a| { a.configure(&self.attr_configuration_context()) .expect("checked attr configuration in constructor") }) } - pub fn get<'a>( - &'a self, - attr: &str, - opts: AttrInspectOptions, - ) -> Option> { - self.0.target_node.attr_or_none(attr, opts).map(|v| { + pub fn get(self, attr: &str, opts: AttrInspectOptions) -> Option> { + self.0.get().target_node.attr_or_none(attr, opts).map(|v| { v.configure(&self.attr_configuration_context()) .expect("checked attr configuration in constructor") }) } - pub fn call_stack(&self) -> Option { - match &self.0.target_node { - TargetNodeOrForward::TargetNode(n) => n.call_stack(), - TargetNodeOrForward::Forward(_, n) => n.call_stack(), + pub fn inputs(self) -> impl Iterator + 'a { + struct InputsCollector { + inputs: Vec, } - } + impl ConfiguredAttrTraversal for InputsCollector { + fn dep(&mut self, _dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { + Ok(()) + } - /// Hash the fields that impact how this target is built. - /// Don't do any recursive hashing of the dependencies. - /// Hashes the attributes _after_ configuration, so changing unconfigured branches that - /// are not taken will not change the hash. - pub fn target_hash(&self, state: &mut H) { - self.label().hash(state); - self.rule_type().hash(state); - self.attrs(AttrInspectOptions::All).for_each(|x| { - // We deliberately don't hash the attribute, as if the value being passed to analysis - // stays the same, we don't care if the attribute that generated it changed. - x.name.hash(state); - x.value.hash(state); - }); + fn input(&mut self, path: BuckPathRef) -> anyhow::Result<()> { + self.inputs.push(path.to_cell_path()); + Ok(()) + } + } + let mut traversal = InputsCollector { inputs: Vec::new() }; + for a in self.attrs(AttrInspectOptions::All) { + a.traverse(self.label().pkg(), &mut traversal) + .expect("inputs collector shouldn't return errors"); + } + traversal.inputs.into_iter() } - /// If this node is a forward node, return the target it forwards to. - pub fn forward_target(&self) -> Option<&ConfiguredTargetNode> { - match &self.0.target_node { - TargetNodeOrForward::TargetNode(_) => None, - TargetNodeOrForward::Forward(_, n) => Some(n), + // TODO(cjhopman): switch to for_each_query? + pub fn queries( + self, + ) -> impl Iterator)> + 'a { + struct Traversal { + queries: Vec<(String, ResolvedQueryLiterals)>, } + let mut traversal = Traversal { + queries: Vec::new(), + }; + impl ConfiguredAttrTraversal for Traversal { + fn dep(&mut self, _dep: &ConfiguredProvidersLabel) -> anyhow::Result<()> { + // ignored. + Ok(()) + } + + fn query( + &mut self, + query: &str, + resolved_literals: &ResolvedQueryLiterals, + ) -> anyhow::Result<()> { + self.queries + .push((query.to_owned(), resolved_literals.clone())); + Ok(()) + } + } + + for a in self.attrs(AttrInspectOptions::All) { + // Optimization. + if !a.attr.coercer().0.may_have_queries { + continue; + } + + a.traverse(self.label().pkg(), &mut traversal).unwrap(); + } + traversal.queries.into_iter() } - pub fn uses_plugins(&self) -> &[PluginKind] { - match &self.0.target_node { + pub fn rule_type(self) -> &'a RuleType { + self.0.get().target_node.rule_type() + } + + pub fn execution_platform_resolution(self) -> &'a ExecutionPlatformResolution { + &self.0.get().execution_platform_resolution + } + + pub fn uses_plugins(self) -> &'a [PluginKind] { + match &self.0.get().target_node { TargetNodeOrForward::TargetNode(target_node) => target_node.uses_plugins(), TargetNodeOrForward::Forward(_, _) => &[], } } - pub fn plugin_lists(&self) -> &PluginLists { - &self.0.plugin_lists - } - - fn plugins_as_attr(&self) -> ConfiguredAttr { + fn plugins_as_attr(self) -> ConfiguredAttr { let mut kinds = Vec::new(); for (kind, plugins) in self.plugin_lists().iter_by_kind() { // Using plugin dep here is a bit of an abuse. However, there's no @@ -603,41 +772,12 @@ impl ConfiguredTargetNode { } ConfiguredAttr::Dict(kinds.into_iter().collect()) } -} - -/// The representation of the deps for a ConfiguredTargetNode. Provides the operations we require -/// (iteration, eq, and hash), but guarantees those aren't recursive of the dep nodes' data. -#[derive(Allocative)] -struct ConfiguredTargetNodeDeps(Box<[ConfiguredTargetNode]>); - -impl ConfiguredTargetNodeDeps { - fn iter(&self) -> impl ExactSizeIterator { - self.0.iter() - } -} -/// We only do Arc comparisons here. The rationale is that each ConfiguredTargetNode should only -/// ever exist in one instance in the graph, so if the ptr eq doesn't match, then we don't do a -/// deep comparison. -impl PartialEq for ConfiguredTargetNodeDeps { - fn eq(&self, other: &Self) -> bool { - let it1 = self.iter(); - let it2 = other.iter(); - it1.len() == it2.len() && it1.zip(it2).all(|(x, y)| Arc::ptr_eq(&x.0, &y.0)) + pub fn plugin_lists(self) -> &'a PluginLists { + &self.0.get().plugin_lists } -} -impl Eq for ConfiguredTargetNodeDeps {} - -/// This has historically only hashed the labels. This may or may not be right, because it means -/// two nodes that reference different instances of a given dependency will hash equal, even if -/// the dependency has definitely changed (e.g. because its own deps changed). -impl Hash for ConfiguredTargetNodeDeps { - fn hash(&self, state: &mut H) { - let it = self.0.iter(); - state.write_usize(it.len()); - for node in it { - node.label().hash(state); - } + pub fn buildfile_path(self) -> &'a BuildFilePath { + self.0.get().target_node.buildfile_path() } } diff --git a/app/buck2_node/src/nodes/configured_node_ref.rs b/app/buck2_node/src/nodes/configured_node_ref.rs new file mode 100644 index 0000000000000..3efc81c74ff5a --- /dev/null +++ b/app/buck2_node/src/nodes/configured_node_ref.rs @@ -0,0 +1,84 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::hash::Hash; +use std::hash::Hasher; + +use buck2_query::query::graph::successors::GraphSuccessors; +use dupe::Dupe; +use starlark_map::StarlarkHashValue; + +use crate::nodes::configured::ConfiguredTargetNode; +use crate::nodes::configured::ConfiguredTargetNodeRef; + +#[derive(Debug, Dupe, Copy, Clone)] +pub struct ConfiguredTargetNodeRefNode<'a> { + // TODO(nga): we store hash here, but we also store hash in `dfs_postorder`. This is redundant. + label_hash: StarlarkHashValue, + node: ConfiguredTargetNodeRef<'a>, +} + +impl PartialEq for ConfiguredTargetNodeRefNode<'_> { + #[inline] + fn eq(&self, other: &Self) -> bool { + // If nodes are the same, their labels must be equal. + self.node.ptr_eq(other.node) + // If nodes are not the same, their hashes are likely different, so we store the hash too. + || (self.label_hash == other.label_hash && self.node.label() == other.node.label()) + } +} + +impl Eq for ConfiguredTargetNodeRefNode<'_> {} + +impl Hash for ConfiguredTargetNodeRefNode<'_> { + #[inline] + fn hash(&self, state: &mut H) { + self.label_hash.hash(state) + } +} + +impl<'a> ConfiguredTargetNodeRefNode<'a> { + #[inline] + pub fn new(node: &'a ConfiguredTargetNode) -> Self { + Self::from_ref(node.as_ref()) + } + + #[inline] + pub fn from_ref(node: ConfiguredTargetNodeRef<'a>) -> Self { + ConfiguredTargetNodeRefNode { + node, + label_hash: node.hashed_label().hash(), + } + } + + #[inline] + pub fn as_ref(&self) -> ConfiguredTargetNodeRef<'a> { + self.node + } + + #[inline] + pub fn to_node(&self) -> ConfiguredTargetNode { + self.node.to_owned() + } +} + +pub struct ConfiguredTargetNodeRefNodeDeps; + +impl<'a> GraphSuccessors> for ConfiguredTargetNodeRefNodeDeps { + #[inline] + fn for_each_successor( + &self, + node: &ConfiguredTargetNodeRefNode<'a>, + mut f: impl FnMut(&ConfiguredTargetNodeRefNode<'a>), + ) { + for dep in node.node.deps() { + f(&ConfiguredTargetNodeRefNode::new(dep)); + } + } +} diff --git a/app/buck2_node/src/nodes/configured_node_visit_all_deps.rs b/app/buck2_node/src/nodes/configured_node_visit_all_deps.rs index de1e62814da39..83dda3e18ee0f 100644 --- a/app/buck2_node/src/nodes/configured_node_visit_all_deps.rs +++ b/app/buck2_node/src/nodes/configured_node_visit_all_deps.rs @@ -7,54 +7,20 @@ * of this source tree. */ -use async_trait::async_trait; -use buck2_query::query::traversal::async_fast_depth_first_postorder_traversal; -use buck2_query::query::traversal::AsyncTraversalDelegate; -use buck2_query::query::traversal::ChildVisitor; -use dupe::Dupe; +use buck2_query::query::graph::bfs::bfs_preorder; -use crate::nodes::configured::ConfiguredTargetNode; -use crate::nodes::configured_ref::ConfiguredGraphNodeRef; -use crate::nodes::configured_ref::ConfiguredGraphNodeRefLookup; +use crate::nodes::configured::ConfiguredTargetNodeRef; +use crate::nodes::configured_node_ref::ConfiguredTargetNodeRefNode; +use crate::nodes::configured_node_ref::ConfiguredTargetNodeRefNodeDeps; /// Visit nodes and all dependencies recursively. -pub async fn configured_node_visit_all_deps( - roots: impl IntoIterator, - // TODO(nga): visitor does not need be either `Sync` or `Send`, - // this is artificial limitation of `async_depth_first_postorder_traversal`. - visitor: impl FnMut(ConfiguredTargetNode) -> anyhow::Result<()> + Send + Sync, -) -> anyhow::Result<()> { - // To support package/recursive patterns, we hold the map by package. To support a - // single target name having multiple instances in the universe, we map them to a list of nodes. - struct Delegate { - visitor: F, - } - - #[async_trait] - impl anyhow::Result<()> + Sync + Send> - AsyncTraversalDelegate for Delegate - { - fn visit(&mut self, target: ConfiguredGraphNodeRef) -> anyhow::Result<()> { - (self.visitor)(target.0) - } - - async fn for_each_child( - &mut self, - target: &ConfiguredGraphNodeRef, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - for dep in target.0.deps() { - func.visit(ConfiguredGraphNodeRef(dep.dupe()))?; - } - Ok(()) - } - } - let mut delegate = Delegate { visitor }; - - let roots = roots - .into_iter() - .map(|node| ConfiguredGraphNodeRef(node.dupe())) - .collect::>(); - async_fast_depth_first_postorder_traversal(&ConfiguredGraphNodeRefLookup, &roots, &mut delegate) - .await +pub fn configured_node_visit_all_deps<'a>( + roots: impl IntoIterator>, + mut visitor: impl FnMut(ConfiguredTargetNodeRef<'a>), +) { + bfs_preorder( + roots.into_iter().map(ConfiguredTargetNodeRefNode::from_ref), + ConfiguredTargetNodeRefNodeDeps, + |node| visitor(node.as_ref()), + ) } diff --git a/app/buck2_node/src/nodes/configured_ref.rs b/app/buck2_node/src/nodes/configured_ref.rs index c71a1402bcf68..e55c6c0f871d9 100644 --- a/app/buck2_node/src/nodes/configured_ref.rs +++ b/app/buck2_node/src/nodes/configured_ref.rs @@ -8,44 +8,53 @@ */ use std::borrow::Cow; +use std::ops::Deref; use allocative::Allocative; -use async_trait::async_trait; use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_query::query::environment::LabeledNode; -use buck2_query::query::environment::NodeLabel; use buck2_query::query::environment::QueryTarget; -use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::NodeLookup; +use buck2_query::query::graph::node::LabeledNode; +use buck2_query::query::graph::node::NodeKey; use dupe::Dupe; use ref_cast::RefCast; -use serde::Serializer; use crate::attrs::attr_type::any_matches::AnyMatches; use crate::attrs::configured_attr::ConfiguredAttr; -use crate::attrs::display::AttrDisplayWithContextExt; -use crate::attrs::fmt_context::AttrFmtContext; use crate::attrs::inspect_options::AttrInspectOptions; -use crate::attrs::serialize::AttrSerializeWithContext; use crate::nodes::configured::ConfiguredTargetNode; /// `ConfiguredTargetNode` as both `LabeledNode` and `NodeLabel` and also `QueryTarget`. #[derive(Debug, Dupe, Clone, RefCast, Allocative)] #[repr(C)] -pub struct ConfiguredGraphNodeRef(pub ConfiguredTargetNode); +pub struct ConfiguredGraphNodeRef(ConfiguredTargetNode); -impl NodeLabel for ConfiguredGraphNodeRef { - fn label_for_filter(&self) -> String { - self.0.label().unconfigured().to_string() +impl NodeKey for ConfiguredGraphNodeRef {} + +impl Deref for ConfiguredGraphNodeRef { + type Target = ConfiguredTargetNode; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 } } impl ConfiguredGraphNodeRef { + #[inline] + pub fn new(node: ConfiguredTargetNode) -> Self { + ConfiguredGraphNodeRef(node) + } + pub fn label(&self) -> &ConfiguredTargetLabel { self.0.label() } + + #[inline] + pub fn into_inner(self) -> ConfiguredTargetNode { + self.0 + } } impl std::fmt::Display for ConfiguredGraphNodeRef { @@ -68,7 +77,8 @@ impl Ord for ConfiguredGraphNodeRef { impl PartialEq for ConfiguredGraphNodeRef { fn eq(&self, other: &Self) -> bool { - self.label().eq(other.label()) + // `ptr_eq` is optimization. + self.0.ptr_eq(&other.0) || self.label().eq(other.label()) } } @@ -76,14 +86,14 @@ impl Eq for ConfiguredGraphNodeRef {} impl std::hash::Hash for ConfiguredGraphNodeRef { fn hash(&self, state: &mut H) { - self.label().hash(state) + self.0.hashed_label().hash().hash(state); } } impl LabeledNode for ConfiguredGraphNodeRef { - type NodeRef = ConfiguredGraphNodeRef; + type Key = ConfiguredGraphNodeRef; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { self } } @@ -91,6 +101,10 @@ impl LabeledNode for ConfiguredGraphNodeRef { impl QueryTarget for ConfiguredGraphNodeRef { type Attr<'a> = ConfiguredAttr; + fn label_for_filter(&self) -> String { + self.0.label().unconfigured().to_string() + } + fn rule_type(&self) -> Cow { Cow::Borrowed(self.0.rule_type().name()) } @@ -99,17 +113,16 @@ impl QueryTarget for ConfiguredGraphNodeRef { self.0.buildfile_path() } - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(self.0.deps().map(ConfiguredGraphNodeRef::ref_cast)) + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a { + self.0.deps().map(ConfiguredGraphNodeRef::ref_cast) } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(self.0.exec_deps().map(ConfiguredGraphNodeRef::ref_cast)) + fn exec_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + self.0.exec_deps().map(ConfiguredGraphNodeRef::ref_cast) } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(self.0.target_deps().map(ConfiguredGraphNodeRef::ref_cast)) + fn target_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + self.0.target_deps().map(ConfiguredGraphNodeRef::ref_cast) } fn attr_any_matches( @@ -157,47 +170,4 @@ impl QueryTarget for ConfiguredGraphNodeRef { } Ok(()) } - - fn call_stack(&self) -> Option { - self.0.call_stack() - } - - fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { - format!( - "{:#}", - attr.as_display(&AttrFmtContext { - package: Some(self.0.label().pkg().dupe()), - }) - ) - } - - fn attr_serialize( - &self, - attr: &Self::Attr<'_>, - serializer: S, - ) -> Result { - attr.serialize_with_ctx( - &AttrFmtContext { - package: Some(self.0.label().pkg().dupe()), - }, - serializer, - ) - } -} - -/// Graph lookup implementation for `ConfiguredGraphNodeRef`. -/// The implementation is trivial because `ConfiguredGraphNodeRef` is both node ref and node. -pub struct ConfiguredGraphNodeRefLookup; - -#[async_trait] -impl AsyncNodeLookup for ConfiguredGraphNodeRefLookup { - async fn get(&self, label: &ConfiguredGraphNodeRef) -> anyhow::Result { - Ok(label.dupe()) - } -} - -impl NodeLookup for ConfiguredGraphNodeRefLookup { - fn get(&self, label: &ConfiguredGraphNodeRef) -> anyhow::Result { - Ok(label.dupe()) - } } diff --git a/app/buck2_node/src/nodes/eval_result.rs b/app/buck2_node/src/nodes/eval_result.rs index 3ce92dd0eefb8..20ee42fe72867 100644 --- a/app/buck2_node/src/nodes/eval_result.rs +++ b/app/buck2_node/src/nodes/eval_result.rs @@ -28,6 +28,7 @@ use itertools::Itertools; use crate::nodes::targets_map::TargetsMap; use crate::nodes::unconfigured::TargetNode; +use crate::nodes::unconfigured::TargetNodeRef; use crate::super_package::SuperPackage; #[derive(Debug, buck2_error::Error)] @@ -161,11 +162,11 @@ impl EvaluationResult { &self.super_package } - pub fn get_target<'a>(&'a self, name: &TargetNameRef) -> Option<&'a TargetNode> { + pub fn get_target<'a>(&'a self, name: &TargetNameRef) -> Option> { self.targets.get(name) } - pub fn resolve_target<'a>(&'a self, path: &TargetNameRef) -> anyhow::Result<&'a TargetNode> { + pub fn resolve_target<'a>(&'a self, path: &TargetNameRef) -> anyhow::Result> { self.get_target(path).ok_or_else(|| { MissingTargetError { target: path.to_owned(), @@ -195,7 +196,7 @@ impl EvaluationResult { for target_info in self.targets().values() { label_to_node.insert( (target_info.label().name().to_owned(), T::default()), - target_info.dupe(), + target_info.to_owned(), ); } (label_to_node, None) @@ -207,7 +208,7 @@ impl EvaluationResult { let node = self.get_target(target_name.as_ref()); match node { Some(node) => { - label_to_node.insert((target_name, extra), node.dupe()); + label_to_node.insert((target_name, extra), node.to_owned()); } None => missing_targets .push(TargetLabel::new(self.package(), target_name.as_ref())), diff --git a/app/buck2_node/src/nodes/frontend.rs b/app/buck2_node/src/nodes/frontend.rs index bf4d167ee586b..3c6a51d90ade9 100644 --- a/app/buck2_node/src/nodes/frontend.rs +++ b/app/buck2_node/src/nodes/frontend.rs @@ -122,7 +122,7 @@ impl TargetGraphCalculation for DiceComputations { ) })?; anyhow::Ok(( - res.resolve_target(target.name())?.dupe(), + res.resolve_target(target.name())?.to_owned(), res.super_package().dupe(), )) }) diff --git a/app/buck2_node/src/nodes/mod.rs b/app/buck2_node/src/nodes/mod.rs index 5bd14371a6cbc..2a80dc9f8c5c6 100644 --- a/app/buck2_node/src/nodes/mod.rs +++ b/app/buck2_node/src/nodes/mod.rs @@ -9,6 +9,7 @@ pub mod configured; pub mod configured_frontend; +pub mod configured_node_ref; pub mod configured_node_visit_all_deps; pub mod configured_ref; pub mod eval_result; diff --git a/app/buck2_node/src/nodes/targets_map.rs b/app/buck2_node/src/nodes/targets_map.rs index b948dbf7a4f42..d0457952be017 100644 --- a/app/buck2_node/src/nodes/targets_map.rs +++ b/app/buck2_node/src/nodes/targets_map.rs @@ -20,8 +20,10 @@ use starlark_map::ordered_set; use starlark_map::ordered_set::OrderedSet; use crate::nodes::unconfigured::TargetNode; +use crate::nodes::unconfigured::TargetNodeRef; #[derive(Debug, buck2_error::Error)] +#[buck2(user)] pub enum TargetsMapRecordError { #[error( "Attempted to register target {0} twice, {}", @@ -82,8 +84,8 @@ impl TargetsMap { } #[inline] - pub fn get(&self, name: &TargetNameRef) -> Option<&TargetNode> { - self.map.get(name).map(|NameIndexed(n)| n) + pub fn get<'a>(&'a self, name: &TargetNameRef) -> Option> { + self.map.get(name).map(|NameIndexed(n)| n.as_ref()) } #[inline] @@ -97,8 +99,10 @@ impl TargetsMap { } #[inline] - pub fn iter(&self) -> impl ExactSizeIterator { - self.map.iter().map(|NameIndexed(n)| (n.label().name(), n)) + pub fn iter(&self) -> impl ExactSizeIterator)> { + self.map + .iter() + .map(|NameIndexed(n)| (n.label().name(), n.as_ref())) } #[inline] @@ -112,7 +116,7 @@ impl TargetsMap { } #[inline] - pub fn values(&self) -> impl ExactSizeIterator { + pub fn values(&self) -> impl ExactSizeIterator> { self.iter().map(|(_, v)| v) } diff --git a/app/buck2_node/src/nodes/unconfigured.rs b/app/buck2_node/src/nodes/unconfigured.rs index b271bebfb5ee8..9e8bb78ba2ff5 100644 --- a/app/buck2_node/src/nodes/unconfigured.rs +++ b/app/buck2_node/src/nodes/unconfigured.rs @@ -9,6 +9,7 @@ use std::hash::Hash; use std::hash::Hasher; +use std::ops::Deref; use std::sync::Arc; use allocative::Allocative; @@ -64,8 +65,19 @@ enum TargetNodeError { /// the attribute names and it doesn't store an entry for something that has a default value. All /// that information is contained in the AttributeSpec. This means that to access an attribute we /// need to look at both the attrs held by the TargetNode and the information in the AttributeSpec. -#[derive(Debug, Clone, Dupe, Eq, PartialEq, Hash, Allocative)] -pub struct TargetNode(pub Arc); +#[derive(Debug, Clone, Eq, PartialEq, Hash, Allocative)] +pub struct TargetNode(triomphe::Arc); + +impl Dupe for TargetNode {} + +impl Deref for TargetNode { + type Target = TargetNodeData; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} /// The kind of the rule, denoting where it can be used and how. #[derive(Debug, Copy, Clone, Dupe, Eq, PartialEq, Hash, Allocative)] @@ -100,6 +112,24 @@ pub struct TargetNodeData { call_stack: Option, } +impl TargetNodeData { + pub fn is_toolchain_rule(&self) -> bool { + self.rule.rule_kind == RuleKind::Toolchain + } + + pub fn rule_type(&self) -> &RuleType { + &self.rule.rule_type + } + + pub fn oncall(&self) -> Option<&str> { + self.package.oncall.as_ref().map(|x| x.as_str()) + } + + pub fn call_stack(&self) -> Option { + self.call_stack.as_ref().map(|s| s.to_string()) + } +} + impl TargetNode { pub fn new( rule: Arc, @@ -109,7 +139,7 @@ impl TargetNode { deps_cache: CoercedDeps, call_stack: Option, ) -> TargetNode { - TargetNode(Arc::new(TargetNodeData { + TargetNode(triomphe::Arc::new(TargetNodeData { rule, package, label, @@ -127,12 +157,8 @@ impl TargetNode { self.0.rule.rule_kind == RuleKind::Configuration } - pub fn is_toolchain_rule(&self) -> bool { - self.0.rule.rule_kind == RuleKind::Toolchain - } - pub fn uses_plugins(&self) -> &[PluginKind] { - &self.0.rule.uses_plugins + self.as_ref().uses_plugins() } pub fn get_default_target_platform(&self) -> Option<&TargetLabel> { @@ -152,36 +178,21 @@ impl TargetNode { } } - pub fn rule_type(&self) -> &RuleType { - &self.0.rule.rule_type - } - + #[inline] pub fn buildfile_path(&self) -> &BuildFilePath { - &self.0.package.buildfile_path - } - - fn deps_cache(&self) -> &CoercedDeps { - &self.0.deps_cache + self.as_ref().buildfile_path() } /// Returns all deps for this node that we know about after processing the build file + #[inline] pub fn deps(&self) -> impl Iterator { - let deps_cache = self.deps_cache(); - deps_cache - .deps - .iter() - .chain(deps_cache.transition_deps.iter().map(|(dep, _tr)| dep)) - .chain(deps_cache.exec_deps.iter()) - .chain(deps_cache.toolchain_deps.iter()) - .chain(deps_cache.plugin_deps.iter()) + self.as_ref().deps() } /// Deps which are to be transitioned to other configuration using transition function. + #[inline] pub fn transition_deps(&self) -> impl Iterator)> { - self.deps_cache() - .transition_deps - .iter() - .map(|x| (&x.0, &x.1)) + self.as_ref().transition_deps() } pub fn label(&self) -> &TargetLabel { @@ -189,40 +200,7 @@ impl TargetNode { } pub fn special_attrs(&self) -> impl Iterator { - let typ_attr = CoercedAttr::String(StringLiteral(self.rule_type().name().into())); - let deps_attr = CoercedAttr::List( - self.deps() - .map(|t| CoercedAttr::Label(ProvidersLabel::default_for(t.dupe()))) - .collect(), - ); - let package_attr = CoercedAttr::String(StringLiteral(ArcStr::from( - self.buildfile_path().to_string(), - ))); - vec![ - (TYPE, typ_attr), - ( - CONFIGURATION_DEPS, - CoercedAttr::List( - self.get_configuration_deps() - .map(|t| CoercedAttr::ConfigurationDep(t.dupe())) - .collect(), - ), - ), - (DEPS, deps_attr), - (PACKAGE, package_attr), - ( - ONCALL, - match self.oncall() { - None => CoercedAttr::None, - Some(x) => CoercedAttr::String(StringLiteral(ArcStr::from(x))), - }, - ), - ] - .into_iter() - } - - pub fn oncall(&self) -> Option<&str> { - self.0.package.oncall.as_ref().map(|x| x.as_str()) + self.as_ref().special_attrs() } pub fn visibility(&self) -> anyhow::Result<&VisibilitySpecification> { @@ -251,48 +229,52 @@ impl TargetNode { Ok(self.visibility()?.0.matches_target(target)) } + #[inline] pub fn attrs(&self, opts: AttrInspectOptions) -> impl Iterator { - self.0.rule.attributes.attrs(&self.0.attributes, opts) + self.as_ref().attrs(opts) } + #[inline] pub fn platform_deps(&self) -> impl Iterator { - self.deps_cache().platform_deps.iter() + self.as_ref().platform_deps() } /// Return `None` if attribute is not present or unknown. + #[inline] pub fn attr_or_none<'a>( &'a self, key: &str, opts: AttrInspectOptions, ) -> Option> { - self.0 - .rule - .attributes - .attr_or_none(&self.0.attributes, key, opts) + self.as_ref().attr_or_none(key, opts) } /// Get attribute. /// /// * `None` if attribute is known but not set and no default. /// * error if attribute is unknown. + #[inline] pub fn attr( &self, key: &str, opts: AttrInspectOptions, ) -> anyhow::Result> { - self.0.rule.attributes.attr(&self.0.attributes, key, opts) + self.as_ref().attr(key, opts) } + #[inline] pub fn target_deps(&self) -> impl Iterator { - self.deps_cache().deps.iter() + self.as_ref().target_deps() } + #[inline] pub fn exec_deps(&self) -> impl Iterator { - self.deps_cache().exec_deps.iter() + self.as_ref().exec_deps() } + #[inline] pub fn get_configuration_deps(&self) -> impl Iterator { - self.deps_cache().configuration_deps.iter() + self.as_ref().get_configuration_deps() } pub fn tests(&self) -> impl Iterator { @@ -366,6 +348,180 @@ impl TargetNode { } pub fn inputs(&self) -> impl Iterator + '_ { + self.as_ref().inputs() + } + + /// Hash the fields that impact how this target is built. + /// Don't do any recursive hashing of the dependencies. + pub fn target_hash(&self, state: &mut H) { + self.label().hash(state); + self.rule_type().hash(state); + self.attrs(AttrInspectOptions::All).for_each(|x| { + // We deliberately don't hash the attribute, as if the value being passed to analysis + // stays the same, we don't care if the attribute that generated it changed. + x.name.hash(state); + x.value.hash(state); + }); + } + + #[inline] + pub fn metadata(&self) -> anyhow::Result> { + self.as_ref().metadata() + } + + #[inline] + pub fn as_ref(&self) -> TargetNodeRef<'_> { + TargetNodeRef(triomphe::Arc::borrow_arc(&self.0)) + } +} + +#[derive(Copy, Clone)] +pub struct TargetNodeRef<'a>(triomphe::ArcBorrow<'a, TargetNodeData>); + +impl<'a> Dupe for TargetNodeRef<'a> {} + +impl<'a> Deref for TargetNodeRef<'a> { + type Target = TargetNodeData; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> TargetNodeRef<'a> { + #[inline] + pub fn label(self) -> &'a TargetLabel { + &self.0.get().label + } + + #[inline] + pub fn to_owned(self) -> TargetNode { + TargetNode(triomphe::ArcBorrow::clone_arc(&self.0)) + } + + pub fn buildfile_path(self) -> &'a BuildFilePath { + &self.0.get().package.buildfile_path + } + + /// Get attribute. + /// + /// * `None` if attribute is known but not set and no default. + /// * error if attribute is unknown. + pub fn attr( + self, + key: &str, + opts: AttrInspectOptions, + ) -> anyhow::Result> { + self.0 + .get() + .rule + .attributes + .attr(&self.0.get().attributes, key, opts) + } + + /// Return `None` if attribute is not present or unknown. + pub fn attr_or_none(&self, key: &str, opts: AttrInspectOptions) -> Option> { + self.0 + .get() + .rule + .attributes + .attr_or_none(&self.0.get().attributes, key, opts) + } + + pub fn attrs(self, opts: AttrInspectOptions) -> impl Iterator> { + self.0 + .get() + .rule + .attributes + .attrs(&self.0.get().attributes, opts) + } + + pub fn special_attrs(self) -> impl Iterator + 'a { + let typ_attr = CoercedAttr::String(StringLiteral(self.rule_type().name().into())); + let deps_attr = CoercedAttr::List( + self.deps() + .map(|t| CoercedAttr::Label(ProvidersLabel::default_for(t.dupe()))) + .collect(), + ); + let package_attr = CoercedAttr::String(StringLiteral(ArcStr::from( + self.buildfile_path().to_string().as_str(), + ))); + vec![ + (TYPE, typ_attr), + ( + CONFIGURATION_DEPS, + CoercedAttr::List( + self.get_configuration_deps() + .map(|t| CoercedAttr::ConfigurationDep(t.dupe())) + .collect(), + ), + ), + (DEPS, deps_attr), + (PACKAGE, package_attr), + ( + ONCALL, + match self.oncall() { + None => CoercedAttr::None, + Some(x) => CoercedAttr::String(StringLiteral(ArcStr::from(x))), + }, + ), + ] + .into_iter() + } + + pub fn metadata(self) -> anyhow::Result> { + self.attr_or_none(METADATA_ATTRIBUTE_FIELD, AttrInspectOptions::All) + .map(|attr| match attr.value { + CoercedAttr::Metadata(m) => Ok(m), + x => Err(TargetNodeError::IncorrectMetadataAttribute(format!("{:?}", x)).into()), + }) + .transpose() + } + + pub fn target_deps(self) -> impl Iterator { + self.0.get().deps_cache.deps.iter() + } + + pub fn exec_deps(self) -> impl Iterator { + self.0.get().deps_cache.exec_deps.iter() + } + + pub fn get_configuration_deps(self) -> impl Iterator { + self.0.get().deps_cache.configuration_deps.iter() + } + + pub fn platform_deps(self) -> impl Iterator { + self.0.get().deps_cache.platform_deps.iter() + } + + /// Returns all deps for this node that we know about after processing the build file + pub fn deps(self) -> impl Iterator { + let deps_cache = &self.0.get().deps_cache; + deps_cache + .deps + .iter() + .chain(deps_cache.transition_deps.iter().map(|(dep, _tr)| dep)) + .chain(deps_cache.exec_deps.iter()) + .chain(deps_cache.toolchain_deps.iter()) + .chain(deps_cache.plugin_deps.iter()) + } + + /// Deps which are to be transitioned to other configuration using transition function. + pub fn transition_deps(self) -> impl Iterator)> { + self.0 + .get() + .deps_cache + .transition_deps + .iter() + .map(|x| (&x.0, &x.1)) + } + + pub fn uses_plugins(self) -> &'a [PluginKind] { + &self.0.get().rule.uses_plugins + } + + pub fn inputs(self) -> impl Iterator + 'a { struct InputsCollector { inputs: Vec, } @@ -428,32 +584,6 @@ impl TargetNode { traversal.inputs.into_iter() } - - pub fn call_stack(&self) -> Option { - self.0.call_stack.as_ref().map(|s| s.to_string()) - } - - /// Hash the fields that impact how this target is built. - /// Don't do any recursive hashing of the dependencies. - pub fn target_hash(&self, state: &mut H) { - self.label().hash(state); - self.rule_type().hash(state); - self.attrs(AttrInspectOptions::All).for_each(|x| { - // We deliberately don't hash the attribute, as if the value being passed to analysis - // stays the same, we don't care if the attribute that generated it changed. - x.name.hash(state); - x.value.hash(state); - }); - } - - pub fn metadata(&self) -> anyhow::Result> { - self.attr_or_none(METADATA_ATTRIBUTE_FIELD, AttrInspectOptions::All) - .map(|attr| match attr.value { - CoercedAttr::Metadata(m) => Ok(m), - x => Err(TargetNodeError::IncorrectMetadataAttribute(format!("{:?}", x)).into()), - }) - .transpose() - } } pub mod testing { diff --git a/app/buck2_node/src/query/configured.rs b/app/buck2_node/src/query/configured.rs index d61c4438c39c1..f396ba570548e 100644 --- a/app/buck2_node/src/query/configured.rs +++ b/app/buck2_node/src/query/configured.rs @@ -12,30 +12,36 @@ use std::borrow::Cow; use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_query::query::environment::LabeledNode; use buck2_query::query::environment::QueryTarget; +use buck2_query::query::graph::node::LabeledNode; use dupe::Dupe; -use serde::Serializer; +use starlark_map::Hashed; use crate::attrs::attr_type::any_matches::AnyMatches; use crate::attrs::configured_attr::ConfiguredAttr; -use crate::attrs::display::AttrDisplayWithContextExt; -use crate::attrs::fmt_context::AttrFmtContext; use crate::attrs::inspect_options::AttrInspectOptions; -use crate::attrs::serialize::AttrSerializeWithContext; use crate::nodes::configured::ConfiguredTargetNode; +use crate::nodes::configured::ConfiguredTargetNodeRef; impl LabeledNode for ConfiguredTargetNode { - type NodeRef = ConfiguredTargetLabel; + type Key = ConfiguredTargetLabel; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { ConfiguredTargetNode::label(self) } + + fn hashed_node_key(&self) -> Hashed<&Self::Key> { + ConfiguredTargetNode::hashed_label(self) + } } impl QueryTarget for ConfiguredTargetNode { type Attr<'a> = ConfiguredAttr; + fn label_for_filter(&self) -> String { + return self.label().unconfigured().to_string(); + } + fn rule_type(&self) -> Cow { Cow::Borrowed(ConfiguredTargetNode::rule_type(self).name()) } @@ -44,21 +50,20 @@ impl QueryTarget for ConfiguredTargetNode { ConfiguredTargetNode::buildfile_path(self) } - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(ConfiguredTargetNode::deps(self).map(|v| v.label())) + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a { + ConfiguredTargetNode::deps(self).map(|v| v.label()) } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(ConfiguredTargetNode::exec_deps(self).map(|v| v.label())) + fn exec_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + ConfiguredTargetNode::exec_deps(self).map(|v| v.label()) } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(ConfiguredTargetNode::target_deps(self).map(|v| v.label())) + fn target_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + ConfiguredTargetNode::target_deps(self).map(|v| v.label()) } - fn tests<'a>(&'a self) -> Option + Send + 'a>> { - Some(Box::new(self.tests().map(|t| t.target().dupe()))) + fn tests<'a>(&'a self) -> Option + Send + 'a> { + Some(self.tests().map(|t| t.target().dupe())) } fn special_attrs_for_each) -> Result<(), E>>( @@ -105,30 +110,16 @@ impl QueryTarget for ConfiguredTargetNode { } Ok(()) } +} - fn call_stack(&self) -> Option { - self.call_stack() - } +impl<'a> LabeledNode for ConfiguredTargetNodeRef<'a> { + type Key = ConfiguredTargetLabel; - fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { - format!( - "{:#}", - attr.as_display(&AttrFmtContext { - package: Some(self.label().pkg().dupe()), - }) - ) + fn node_key(&self) -> &Self::Key { + ConfiguredTargetNodeRef::label(*self) } - fn attr_serialize( - &self, - attr: &Self::Attr<'_>, - serializer: S, - ) -> Result { - attr.serialize_with_ctx( - &AttrFmtContext { - package: Some(self.label().pkg().dupe()), - }, - serializer, - ) + fn hashed_node_key(&self) -> Hashed<&Self::Key> { + ConfiguredTargetNodeRef::hashed_label(*self) } } diff --git a/app/buck2_node/src/query/unconfigured.rs b/app/buck2_node/src/query/unconfigured.rs index 5d50071b4017f..d64193cfda2f8 100644 --- a/app/buck2_node/src/query/unconfigured.rs +++ b/app/buck2_node/src/query/unconfigured.rs @@ -12,22 +12,19 @@ use std::borrow::Cow; use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::target::label::TargetLabel; -use buck2_query::query::environment::LabeledNode; use buck2_query::query::environment::QueryTarget; +use buck2_query::query::graph::node::LabeledNode; use dupe::Dupe; -use serde::Serializer; use crate::attrs::coerced_attr::CoercedAttr; -use crate::attrs::display::AttrDisplayWithContextExt; -use crate::attrs::fmt_context::AttrFmtContext; use crate::attrs::inspect_options::AttrInspectOptions; -use crate::attrs::serialize::AttrSerializeWithContext; use crate::nodes::unconfigured::TargetNode; +use crate::nodes::unconfigured::TargetNodeData; impl LabeledNode for TargetNode { - type NodeRef = TargetLabel; + type Key = TargetLabel; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { TargetNode::label(self) } } @@ -36,28 +33,27 @@ impl QueryTarget for TargetNode { type Attr<'a> = CoercedAttr; fn rule_type(&self) -> Cow { - Cow::Borrowed(TargetNode::rule_type(self).name()) + Cow::Borrowed(TargetNodeData::rule_type(self).name()) } fn buildfile_path(&self) -> &BuildFilePath { TargetNode::buildfile_path(self) } - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(TargetNode::deps(self)) + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a { + TargetNode::deps(self) } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(TargetNode::exec_deps(self)) + fn exec_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + TargetNode::exec_deps(self) } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(TargetNode::target_deps(self)) + fn target_deps<'a>(&'a self) -> impl Iterator + Send + 'a { + TargetNode::target_deps(self) } - fn tests<'a>(&'a self) -> Option + Send + 'a>> { - Some(Box::new(self.tests().map(|t| t.target().dupe()))) + fn tests<'a>(&'a self) -> Option + Send + 'a> { + Some(self.tests().map(|t| t.target().dupe())) } fn attr_any_matches( @@ -104,30 +100,4 @@ impl QueryTarget for TargetNode { } Ok(()) } - - fn call_stack(&self) -> Option { - self.call_stack() - } - - fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { - format!( - "{:#}", - attr.as_display(&AttrFmtContext { - package: Some(self.label().pkg().dupe()), - }) - ) - } - - fn attr_serialize( - &self, - attr: &Self::Attr<'_>, - serializer: S, - ) -> Result { - attr.serialize_with_ctx( - &AttrFmtContext { - package: Some(self.label().pkg().dupe()), - }, - serializer, - ) - } } diff --git a/app/buck2_node/src/target_calculation.rs b/app/buck2_node/src/target_calculation.rs index 1d1d225a537a2..5a4bb34af9a6d 100644 --- a/app/buck2_node/src/target_calculation.rs +++ b/app/buck2_node/src/target_calculation.rs @@ -8,6 +8,7 @@ */ use async_trait::async_trait; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersLabel; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; @@ -21,7 +22,7 @@ pub trait ConfiguredTargetCalculationImpl: Send + Sync + 'static { &self, ctx: &DiceComputations, target: &TargetLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result; } @@ -45,13 +46,13 @@ pub trait ConfiguredTargetCalculation { async fn get_configured_target( &self, target: &TargetLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result; async fn get_configured_provider_label( &self, target: &ProvidersLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result; async fn get_default_configured_target( @@ -65,22 +66,22 @@ impl ConfiguredTargetCalculation for DiceComputations { async fn get_configured_target( &self, target: &TargetLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result { CONFIGURED_TARGET_CALCULATION .get()? - .get_configured_target(self, target, global_target_platform) + .get_configured_target(self, target, global_cfg_options) .await } async fn get_configured_provider_label( &self, target: &ProvidersLabel, - global_target_platform: Option<&TargetLabel>, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result { let configured_target_label = CONFIGURED_TARGET_CALCULATION .get()? - .get_configured_target(self, target.target(), global_target_platform) + .get_configured_target(self, target.target(), global_cfg_options) .await?; Ok(ConfiguredProvidersLabel::new( configured_target_label, @@ -94,7 +95,7 @@ impl ConfiguredTargetCalculation for DiceComputations { ) -> anyhow::Result { CONFIGURED_TARGET_CALCULATION .get()? - .get_configured_target(self, target, None) + .get_configured_target(self, target, &GlobalCfgOptions::default()) .await } } diff --git a/app/buck2_node/src/visibility.rs b/app/buck2_node/src/visibility.rs index 3814d572dd55f..71787a34bdd2b 100644 --- a/app/buck2_node/src/visibility.rs +++ b/app/buck2_node/src/visibility.rs @@ -26,7 +26,7 @@ pub enum VisibilityError { #[error( "`{0}` is not visible to `{1}` (run `buck2 uquery --output-attribute visibility {0}` to check the visibility)" )] - #[buck2(user)] + #[buck2(user, tag = Visibility)] NotVisibleTo(TargetLabel, TargetLabel), } diff --git a/app/buck2_query/BUCK b/app/buck2_query/BUCK index 5ad258664c2b7..347a03f0fc3ab 100644 --- a/app/buck2_query/BUCK +++ b/app/buck2_query/BUCK @@ -17,7 +17,6 @@ rust_library( "fbsource//third-party/rust:indoc", "fbsource//third-party/rust:itertools", "fbsource//third-party/rust:ref-cast", - "fbsource//third-party/rust:serde", "fbsource//third-party/rust:tokio", "//buck2/allocative/allocative:allocative", "//buck2/app/buck2_core:buck2_core", diff --git a/app/buck2_query/Cargo.toml b/app/buck2_query/Cargo.toml index 73c4c3f11e4fa..e79cc66188300 100644 --- a/app/buck2_query/Cargo.toml +++ b/app/buck2_query/Cargo.toml @@ -17,7 +17,6 @@ indexmap = { workspace = true } indoc = { workspace = true } itertools = { workspace = true } ref-cast = { workspace = true } -serde = { workspace = true } tokio = { workspace = true } allocative = { workspace = true } diff --git a/app/buck2_query/src/query/buck_types.rs b/app/buck2_query/src/query/buck_types.rs index ee158cf61e621..b60487a243473 100644 --- a/app/buck2_query/src/query/buck_types.rs +++ b/app/buck2_query/src/query/buck_types.rs @@ -10,25 +10,8 @@ use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_core::target::label::TargetLabel; -use crate::query::environment::ConfiguredOrUnconfiguredTargetLabel; -use crate::query::environment::NodeLabel; +use crate::query::graph::node::NodeKey; -impl ConfiguredOrUnconfiguredTargetLabel for TargetLabel { - fn unconfigured_label(&self) -> &TargetLabel { - self - } -} +impl NodeKey for TargetLabel {} -impl NodeLabel for TargetLabel {} - -impl ConfiguredOrUnconfiguredTargetLabel for ConfiguredTargetLabel { - fn unconfigured_label(&self) -> &TargetLabel { - self.unconfigured() - } -} - -impl NodeLabel for ConfiguredTargetLabel { - fn label_for_filter(&self) -> String { - return self.unconfigured().to_string(); - } -} +impl NodeKey for ConfiguredTargetLabel {} diff --git a/app/buck2_query/src/query/environment/mod.rs b/app/buck2_query/src/query/environment/mod.rs index eabd0413d42a7..11ac1e6eca5eb 100644 --- a/app/buck2_query/src/query/environment/mod.rs +++ b/app/buck2_query/src/query/environment/mod.rs @@ -8,10 +8,8 @@ */ use std::borrow::Cow; -use std::collections::HashMap; use std::fmt::Debug; -use std::fmt::Display; -use std::hash::Hash; +use std::iter; use anyhow::Context; use async_trait::async_trait; @@ -19,17 +17,21 @@ use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::package::PackageLabel; -use buck2_core::target::label::TargetLabel; use dupe::Dupe; +use dupe::OptionDupedExt; use futures::stream::FuturesUnordered; use futures::stream::TryStreamExt; -use starlark_map::ordered_set::OrderedSet; +use crate::query::graph::async_bfs::async_bfs_find_path; +use crate::query::graph::graph::Graph; +use crate::query::graph::node::LabeledNode; +use crate::query::graph::node::NodeKey; +use crate::query::graph::successors::AsyncChildVisitor; +use crate::query::graph::successors::GraphSuccessors; use crate::query::syntax::simple::eval::error::QueryError; use crate::query::syntax::simple::eval::file_set::FileSet; use crate::query::syntax::simple::eval::set::TargetSet; -use crate::query::syntax::simple::eval::set::TargetSetExt; -use crate::query::traversal::AsyncTraversalDelegate; +use crate::query::traversal::AsyncNodeLookup; use crate::query::traversal::ChildVisitor; mod tests; @@ -42,7 +44,7 @@ pub enum QueryEnvironmentError { } impl QueryEnvironmentError { - pub fn missing_target, Iter: IntoIterator>( + pub fn missing_target, Iter: IntoIterator>( target: &T, package_targets: Iter, ) -> Self { @@ -54,23 +56,6 @@ impl QueryEnvironmentError { } } -pub trait NodeLabel: Clone + Hash + PartialEq + Eq + Debug + Display + Send + Sync { - /// `filter()` function will use this. - fn label_for_filter(&self) -> String { - self.to_string() - } -} - -pub trait ConfiguredOrUnconfiguredTargetLabel: NodeLabel { - fn unconfigured_label(&self) -> &TargetLabel; -} - -pub trait LabeledNode: Dupe + Send + Sync + 'static { - type NodeRef: NodeLabel; - - fn node_ref(&self) -> &Self::NodeRef; -} - pub struct QueryTargets {} impl QueryTargets { @@ -90,6 +75,11 @@ impl QueryTargets { pub trait QueryTarget: LabeledNode + Dupe + Send + Sync + 'static { type Attr<'a>: ?Sized + Debug + 'a; + /// `filter()` function uses this. + fn label_for_filter(&self) -> String { + self.node_key().to_string() + } + /// Returns the input files for this node. fn inputs_for_each Result<(), E>>(&self, func: F) -> Result<(), E>; @@ -98,27 +88,16 @@ pub trait QueryTarget: LabeledNode + Dupe + Send + Sync + 'static { /// Return the path to the buildfile that defines this target, e.g. `fbcode//foo/bar/TARGETS` fn buildfile_path(&self) -> &BuildFilePath; - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn deps<'a>(&'a self) -> Box + Send + 'a>; + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a; - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn exec_deps<'a>(&'a self) -> Box + Send + 'a>; + fn exec_deps<'a>(&'a self) -> impl Iterator + Send + 'a; - // TODO(cjhopman): Use existential traits to remove the Box<> once they are stabilized. - fn target_deps<'a>(&'a self) -> Box + Send + 'a>; + fn target_deps<'a>(&'a self) -> impl Iterator + Send + 'a; - fn tests<'a>(&'a self) -> Option + Send + 'a>> { - None + fn tests<'a>(&'a self) -> Option + Send + 'a> { + None::> } - fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String; - - fn attr_serialize( - &self, - attr: &Self::Attr<'_>, - serializer: S, - ) -> Result; - fn attr_any_matches( attr: &Self::Attr<'_>, filter: &dyn Fn(&str) -> anyhow::Result, @@ -135,8 +114,6 @@ pub trait QueryTarget: LabeledNode + Dupe + Send + Sync + 'static { ) -> Result<(), E>; fn map_attr>) -> R>(&self, key: &str, func: F) -> R; - - fn call_stack(&self) -> Option; } #[async_trait] @@ -153,12 +130,12 @@ pub trait QueryEnvironment: Send + Sync { async fn get_node( &self, - node_ref: &::NodeRef, + node_ref: &::Key, ) -> anyhow::Result; async fn get_node_for_default_configured_target( &self, - node_ref: &::NodeRef, + node_ref: &::Key, ) -> anyhow::Result>; /// Evaluates a literal target pattern. See buck2_common::pattern @@ -172,13 +149,15 @@ pub trait QueryEnvironment: Send + Sync { async fn dfs_postorder( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + successors: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()>; async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + successors: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()>; @@ -190,65 +169,25 @@ pub trait QueryEnvironment: Send + Sync { self.rdeps(from, to, None).await } + #[allow(clippy::from_iter_instead_of_collect)] async fn somepath( &self, from: &TargetSet, to: &TargetSet, ) -> anyhow::Result> { - struct Delegate<'a, Q: QueryTarget> { - to: &'a TargetSet, - /// Contains targets that were reached starting from `from` that have a path to `to`. - path: TargetSet, - } - - #[async_trait] - impl<'a, Q: QueryTarget> AsyncTraversalDelegate for Delegate<'a, Q> { - fn visit(&mut self, target: Q) -> anyhow::Result<()> { - // NOTE: It would be better to just only post-order visit our parents, but that is - // not possible because we push *all* children when visiting a node, so we will not - // just post-visit all parents when we interrupt the search. - // NOTE: We assert! around the insertions below because we know each node should - // only be post-visited once but since we rely on `last()`, it matters so we check - // it. - - if let Some(head) = self.path.last() { - if target.deps().any(|t| t == head.node_ref()) { - assert!(self.path.insert(target)); - } - return Ok(()); - } - - if self.to.contains(target.node_ref()) { - assert!(self.path.insert(target)); - } - - Ok(()) - } - - async fn for_each_child( - &mut self, - target: &Q, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - // Stop adding more children if we are putting a path back together. - if !self.path.is_empty() || self.to.contains(target.node_ref()) { - return Ok(()); - } - let res: anyhow::Result<_> = try { - for dep in target.deps() { - func.visit(dep.clone())?; - } - }; - res.with_context(|| format!("Error traversing children of `{}`", target.node_ref())) - } - } - - let mut delegate = Delegate { - path: TargetSet::new(), - to, - }; - self.dfs_postorder(from, &mut delegate).await?; - Ok(delegate.path) + let mut path = async_bfs_find_path( + from.iter(), + QueryEnvironmentAsNodeLookup { env: self }, + QueryTargetDepsSuccessors, + |t| to.get(t).duped(), + ) + .await? + .unwrap_or_default(); + + path.reverse(); + + let target_set = TargetSet::from_iter(path); + Ok(target_set) } async fn allbuildfiles(&self, _universe: &TargetSet) -> anyhow::Result { @@ -269,99 +208,47 @@ pub trait QueryEnvironment: Send + Sync { from: &TargetSet, depth: Option, ) -> anyhow::Result> { - // First, we map all deps to their rdeps (parents). - // This effectively allows traversing the graph later, in reverse (following dependency back-edges). - struct ParentsCollectorDelegate { - parents: HashMap>, - // Keep track of nodes in-universe so that, if any rdeps are collected out-of-universe, - // we don't return them. - nodes_in_universe: TargetSet, - } - - #[async_trait] - impl AsyncTraversalDelegate for ParentsCollectorDelegate { - fn visit(&mut self, target: Q) -> anyhow::Result<()> { - self.nodes_in_universe.insert(target); - Ok(()) - } - - async fn for_each_child( - &mut self, - target: &Q, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - for dep in target.deps() { - func.visit(dep.clone()).with_context(|| { - format!("Error traversing children of `{}`", target.node_ref()) - })?; - self.parents - .entry(dep.clone()) - .or_default() - .insert(target.node_ref().clone()); - } - Ok(()) - } - } - - let mut parents_collector_delegate = ParentsCollectorDelegate { - parents: HashMap::new(), - nodes_in_universe: TargetSet::new(), - }; - - self.dfs_postorder(universe, &mut parents_collector_delegate) - .await?; - - // Now that we have a mapping of back-edges, traverse deps graph in reverse. - struct ReverseDelegate { - rdeps: TargetSet, - parents: HashMap>, - } + let graph = Graph::build_stable_dfs( + &QueryEnvironmentAsNodeLookup { env: self }, + universe.iter().map(|n| n.node_key().clone()), + QueryTargetDepsSuccessors, + ) + .await?; - #[async_trait] - impl AsyncTraversalDelegate for ReverseDelegate { - fn visit(&mut self, target: Q) -> anyhow::Result<()> { - self.rdeps.insert(target); - Ok(()) - } + let graph = graph.reverse(); - async fn for_each_child( - &mut self, - target: &Q, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - if let Some(parents) = self.parents.get(target.node_ref()) { - for parent in parents { - func.visit(parent.clone()).with_context(|| { - format!("Error traversing parents of `{}`", target.node_ref()) - })?; - } - } - Ok(()) - } - } + let mut rdeps = TargetSet::new(); - let mut delegate = ReverseDelegate { - rdeps: TargetSet::new(), - parents: parents_collector_delegate.parents, + let mut visit = |target| { + rdeps.insert_unique_unchecked(target); + Ok(()) }; - let roots_in_universe = from.intersect(&parents_collector_delegate.nodes_in_universe)?; + let roots_in_universe = from.filter(|t| Ok(graph.get(t.node_key()).is_some()))?; match depth { // For unbounded traversals, buck1 recommends specifying a large value. We'll accept either a negative (like -1) or // a large value as unbounded. We can't just call it optional because args are positional only in the query syntax // and so to specify a filter you need to specify a depth. Some(v) if (0..1_000_000_000).contains(&v) => { - self.depth_limited_traversal(&roots_in_universe, &mut delegate, v as u32) - .await?; + let graph = graph.take_max_depth( + roots_in_universe.iter().map(|t| t.node_key().clone()), + v as u32, + ); + graph.depth_first_postorder_traversal( + roots_in_universe.iter().map(|t| t.node_key().clone()), + |t| visit(t.clone()), + )?; } _ => { - self.dfs_postorder(&roots_in_universe, &mut delegate) - .await?; + graph.depth_first_postorder_traversal( + roots_in_universe.iter().map(|t| t.node_key().clone()), + |t| visit(t.clone()), + )?; } } - Ok(delegate.rdeps) + Ok(rdeps) } async fn testsof( @@ -386,7 +273,7 @@ pub trait QueryEnvironment: Send + Sync { let test = self.get_node(&test).await.with_context(|| { format!( "Error getting test of target {}", - LabeledNode::node_ref(target), + LabeledNode::node_key(target), ) })?; anyhow::Ok(test) @@ -427,7 +314,7 @@ pub trait QueryEnvironment: Send + Sync { .with_context(|| { format!( "Error getting test of target {}", - LabeledNode::node_ref(target), + LabeledNode::node_key(target), ) })?; anyhow::Ok(test) @@ -464,37 +351,35 @@ pub async fn deps( let mut deps = TargetSet::new(); struct Delegate<'a, Q: QueryTarget> { - deps: &'a mut TargetSet, filter: Option<&'a dyn TraversalFilter>, } - #[async_trait] - impl<'a, Q: QueryTarget> AsyncTraversalDelegate for Delegate<'a, Q> { - fn visit(&mut self, target: Q) -> anyhow::Result<()> { - self.deps.insert(target); - Ok(()) - } + let visit = |target| { + deps.insert_unique_unchecked(target); + Ok(()) + }; + impl<'a, Q: QueryTarget> AsyncChildVisitor for Delegate<'a, Q> { async fn for_each_child( - &mut self, + &self, target: &Q, - func: &mut dyn ChildVisitor, + mut func: impl ChildVisitor, ) -> anyhow::Result<()> { let res: anyhow::Result<_> = try { match self.filter { Some(filter) => { for dep in filter.get_children(target).await?.iter() { - func.visit(dep.node_ref().clone())?; + func.visit(dep.node_key())?; } } None => { for dep in target.deps() { - func.visit(dep.clone())?; + func.visit(dep)?; } } } }; - res.with_context(|| format!("Error traversing children of `{}`", target.node_ref())) + res.with_context(|| format!("Error traversing children of `{}`", target.node_key())) } } @@ -503,27 +388,53 @@ pub async fn deps( // a large value as unbounded. We can't just call it optional because args are positional only in the query syntax // and so to specify a filter you need to specify a depth. Some(v) if (0..1_000_000_000).contains(&v) => { - env.depth_limited_traversal( - targets, - &mut Delegate { - deps: &mut deps, - filter, - }, - v as u32, - ) - .await?; + env.depth_limited_traversal(targets, Delegate { filter }, visit, v as u32) + .await?; } _ => { - env.dfs_postorder( - targets, - &mut Delegate { - deps: &mut deps, - filter, - }, - ) - .await?; + env.dfs_postorder(targets, Delegate { filter }, visit) + .await?; } } Ok(deps) } + +pub struct QueryTargetDepsSuccessors; + +impl AsyncChildVisitor for QueryTargetDepsSuccessors { + async fn for_each_child( + &self, + node: &T, + mut children: impl ChildVisitor, + ) -> anyhow::Result<()> { + for dep in node.deps() { + children.visit(dep)?; + } + Ok(()) + } +} + +impl GraphSuccessors for QueryTargetDepsSuccessors +where + T: QueryTarget, +{ + fn for_each_successor(&self, node: &T, mut cb: impl FnMut(&T)) { + for dep in node.deps() { + cb(dep); + } + } +} + +pub struct QueryEnvironmentAsNodeLookup<'q, Q: QueryEnvironment + ?Sized> { + pub env: &'q Q, +} + +#[async_trait] +impl<'q, Q: QueryEnvironment + ?Sized> AsyncNodeLookup + for QueryEnvironmentAsNodeLookup<'q, Q> +{ + async fn get(&self, label: &::Key) -> anyhow::Result { + self.env.get_node(label).await + } +} diff --git a/app/buck2_query/src/query/environment/tests.rs b/app/buck2_query/src/query/environment/tests.rs index d679f60ddacab..3a7c89258611b 100644 --- a/app/buck2_query/src/query/environment/tests.rs +++ b/app/buck2_query/src/query/environment/tests.rs @@ -20,7 +20,6 @@ use derive_more::Display; use derive_more::From; use dupe::OptionDupedExt; use indexmap::IndexSet; -use serde::Serializer; use super::*; use crate::query::traversal::AsyncNodeLookup; @@ -28,7 +27,7 @@ use crate::query::traversal::AsyncNodeLookup; #[derive(Debug, Copy, Clone, Dupe, Eq, PartialEq, Hash, Display, From)] struct TestTargetId(u64); -impl NodeLabel for TestTargetId {} +impl NodeKey for TestTargetId {} #[derive(Debug, Copy, Clone, Dupe, Eq, PartialEq, Hash, Display)] struct TestTargetAttr; @@ -47,9 +46,9 @@ impl fmt::Debug for TestTarget { } impl LabeledNode for TestTarget { - type NodeRef = TestTargetId; + type Key = TestTargetId; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { &self.id } } @@ -69,30 +68,18 @@ impl QueryTarget for TestTarget { unimplemented!() } - fn deps<'a>(&'a self) -> Box + Send + 'a> { + fn deps<'a>(&'a self) -> Box + Send + 'a> { Box::new(self.deps.iter()) } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { + fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { Box::new(std::iter::empty()) } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { + fn target_deps<'a>(&'a self) -> Box + Send + 'a> { Box::new(std::iter::empty()) } - fn attr_to_string_alternate(&self, _attr: &Self::Attr<'_>) -> String { - unimplemented!("not needed for tests") - } - - fn attr_serialize( - &self, - _attr: &Self::Attr<'_>, - _serializer: S, - ) -> Result { - unimplemented!("not needed for tests") - } - fn attr_any_matches( _attr: &Self::Attr<'_>, _filter: &dyn Fn(&str) -> anyhow::Result, @@ -117,10 +104,6 @@ impl QueryTarget for TestTarget { fn map_attr>) -> R>(&self, _key: &str, _func: F) -> R { unimplemented!() } - - fn call_stack(&self) -> Option { - None - } } struct TestEnv { @@ -128,7 +111,7 @@ struct TestEnv { } impl NodeLookup for TestEnv { - fn get(&self, label: &::NodeRef) -> anyhow::Result { + fn get(&self, label: &::Key) -> anyhow::Result { self.graph .get(label) .duped() @@ -138,10 +121,7 @@ impl NodeLookup for TestEnv { #[async_trait] impl AsyncNodeLookup for TestEnv { - async fn get( - &self, - label: &::NodeRef, - ) -> anyhow::Result { + async fn get(&self, label: &::Key) -> anyhow::Result { self.graph .get(label) .duped() @@ -155,14 +135,14 @@ impl QueryEnvironment for TestEnv { async fn get_node( &self, - node_ref: &::NodeRef, + node_ref: &::Key, ) -> anyhow::Result { >::get(self, node_ref) } async fn get_node_for_default_configured_target( &self, - _node_ref: &::NodeRef, + _node_ref: &::Key, ) -> anyhow::Result> { unimplemented!() } @@ -178,19 +158,21 @@ impl QueryEnvironment for TestEnv { async fn dfs_postorder( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { // TODO: Should this be part of QueryEnvironment's default impl? - async_depth_first_postorder_traversal(self, root.iter_names(), delegate).await + async_depth_first_postorder_traversal(self, root.iter_names(), delegate, visit).await } async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()> { - async_depth_limited_traversal(self, root.iter_names(), delegate, depth).await + async_depth_limited_traversal(self, root.iter_names(), delegate, visit, depth).await } async fn owner(&self, _paths: &FileSet) -> anyhow::Result> { @@ -276,9 +258,8 @@ async fn test_many_paths() -> anyhow::Result<()> { let expected = env.set("1,10,11,2,3")?; assert_eq!(path, expected); - // We iterate with a stack so this is why we find this path let path = env.somepath(&env.set("1")?, &env.set("3")?).await?; - let expected = env.set("3,11,10,1")?; + let expected = env.set("3,2,1")?; assert_eq!(path, expected); Ok(()) @@ -299,7 +280,7 @@ async fn test_distinct_paths() -> anyhow::Result<()> { // Same as above let path = env.somepath(&env.set("1,2")?, &env.set("100,200")?).await?; - let expected = env.set("200,20,2")?; + let expected = env.set("100,10,1")?; assert_eq!(path, expected); Ok(()) @@ -362,7 +343,39 @@ async fn test_paths_with_cycles_present() -> anyhow::Result<()> { assert_eq!(path, env.set("1,2,3,4,5")?); let path = env.rdeps(&env.set("1")?, &env.set("3")?, Some(2)).await?; - assert_eq!(path, env.set("3,2,4,1")?); + assert_eq!(path, env.set("4,1,2,3")?); + + Ok(()) +} + +#[tokio::test] +async fn test_rdeps() -> anyhow::Result<()> { + let mut env = TestEnvBuilder::default(); + env.edge(1, 2); + env.edge(2, 3); + env.edge(3, 4); // Dead end. + env.edge(4, 5); + env.edge(1, 3); // Shortcut. + env.edge(3, 6); + let env = env.build(); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, Some(0)).await?; + assert_eq!(path, env.set("6")?); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, Some(1)).await?; + assert_eq!(path, env.set("3,6")?); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, Some(2)).await?; + assert_eq!(path, env.set("1,2,3,6")?); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, Some(3)).await?; + assert_eq!(path, env.set("1,2,3,6")?); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, Some(4)).await?; + assert_eq!(path, env.set("1,2,3,6")?); + + let path = env.rdeps(&env.set("1")?, &env.set("6")?, None).await?; + assert_eq!(path, env.set("1,2,3,6")?); Ok(()) } diff --git a/app/buck2_query/src/query/futures_queue_generic.rs b/app/buck2_query/src/query/futures_queue_generic.rs deleted file mode 100644 index 6d79b21766088..0000000000000 --- a/app/buck2_query/src/query/futures_queue_generic.rs +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under both the MIT license found in the - * LICENSE-MIT file in the root directory of this source tree and the Apache - * License, Version 2.0 found in the LICENSE-APACHE file in the root directory - * of this source tree. - */ - -use std::future::Future; -use std::pin::Pin; - -use futures::stream::FuturesOrdered; -use futures::stream::FuturesUnordered; -use futures::Stream; -use futures::StreamExt; - -/// `FuturesOrdered` or `FuturesUnordered`. -// This can be done with GAT, but GAT is unstable, and requires too much work to make it work, -// and this switch has only a little runtime overhead. -pub(crate) enum FuturesQueue { - Ordered(FuturesOrdered), - Unordered(FuturesUnordered), -} - -impl FuturesQueue { - pub(crate) fn new_ordered() -> Self { - FuturesQueue::Ordered(FuturesOrdered::new()) - } - - pub(crate) fn new_unordered() -> Self { - FuturesQueue::Unordered(FuturesUnordered::new()) - } - - pub(crate) fn push(&mut self, fut: Fut) { - match self { - FuturesQueue::Ordered(futures_ordered) => futures_ordered.push_back(fut), - FuturesQueue::Unordered(futures_unordered) => futures_unordered.push(fut), - } - } -} - -impl Stream for FuturesQueue { - type Item = Fut::Output; - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - match self.get_mut() { - FuturesQueue::Ordered(futures_ordered) => futures_ordered.poll_next_unpin(cx), - FuturesQueue::Unordered(futures_unordered) => futures_unordered.poll_next_unpin(cx), - } - } -} diff --git a/app/buck2_query/src/query/graph/async_bfs.rs b/app/buck2_query/src/query/graph/async_bfs.rs new file mode 100644 index 0000000000000..fc83cd4c3f030 --- /dev/null +++ b/app/buck2_query/src/query/graph/async_bfs.rs @@ -0,0 +1,383 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::future; +use std::mem; + +use buck2_error::Context; +use futures::future::Either; +use futures::stream::FuturesOrdered; +use futures::StreamExt; +use starlark_map::unordered_map; +use starlark_map::unordered_map::UnorderedMap; +use starlark_map::Hashed; + +use crate::query::graph::node::LabeledNode; +use crate::query::graph::successors::AsyncChildVisitor; +use crate::query::traversal::AsyncNodeLookup; + +struct Node { + /// `None` for roots. + parent: Option, + /// `None` when not yet looked up. + node: Option, +} + +struct BfsVisited { + visited: UnorderedMap>, +} + +impl BfsVisited { + fn take_path(mut self, last: &N::Key, mut item: impl FnMut(N)) -> anyhow::Result<()> { + let node = self + .visited + .remove(last) + .with_context(|| format!("missing node {} (internal error)", last))?; + if node.node.is_some() { + return Err(anyhow::anyhow!("duplicate node {} (internal error)", last)); + } + let mut parent_key = node.parent; + while let Some(key) = parent_key { + let node = self + .visited + .remove(&key) + .with_context(|| format!("missing node {} (internal error)", key))?; + item( + node.node + .with_context(|| format!("missing node {} (internal error)", key))?, + ); + parent_key = node.parent; + } + Ok(()) + } +} + +pub(crate) async fn async_bfs_find_path<'a, N: LabeledNode + 'static>( + roots: impl IntoIterator, + lookup: impl AsyncNodeLookup, + successors: impl AsyncChildVisitor, + target: impl Fn(&N::Key) -> Option + Sync, +) -> anyhow::Result>> { + let lookup = &lookup; + + let mut visited = BfsVisited:: { + visited: UnorderedMap::new(), + }; + + let mut queue = FuturesOrdered::new(); + + for root in roots { + let root_key = Hashed::new(root.node_key()); + match visited.visited.raw_entry_mut().from_key_hashed(root_key) { + unordered_map::RawEntryMut::Occupied(_) => {} + unordered_map::RawEntryMut::Vacant(e) => { + if let Some(target) = target(root_key.key()) { + return Ok(Some(vec![target])); + } + + e.insert_hashed( + root_key.cloned(), + Node { + parent: None, + node: None, + }, + ); + queue.push_back(Either::Left(future::ready(( + root_key.into_key().clone(), + anyhow::Ok(root.dupe()), + )))); + } + } + } + + while let Some((key, node)) = queue.next().await { + match node { + Ok(node) => { + let mut found: Option = None; + successors + .for_each_child(&node, &mut |succ: &N::Key| { + if found.is_some() { + return Ok(()); + } + + let succ = Hashed::new(succ); + match visited.visited.raw_entry_mut().from_key_hashed(succ) { + unordered_map::RawEntryMut::Occupied(_) => {} + unordered_map::RawEntryMut::Vacant(e) => { + if let Some(target) = target(succ.key()) { + found = Some(target); + return Ok(()); + } + + e.insert_hashed( + succ.cloned(), + Node { + parent: Some(node.node_key().clone()), + node: None, + }, + ); + let succ = (*succ.key()).clone(); + queue.push_back(Either::Right(async move { + let succ_node = lookup.get(&succ).await; + (succ, succ_node) + })); + } + } + + Ok(()) + }) + .await?; + + if let Some(found) = found { + let key = node.node_key().clone(); + let mut path: Vec = vec![found, node]; + visited.take_path(&key, |node| path.push(node))?; + path.reverse(); + return Ok(Some(path)); + } + let prev = mem::replace( + &mut visited + .visited + .get_mut(&key) + .with_context(|| format!("missing node {} (internal error)", key))? + .node, + Some(node), + ); + if prev.is_some() { + return Err(anyhow::anyhow!("duplicate node {} (internal error)", key)); + } + } + Err(mut e) => { + e = e.context(format!("traversing {}", key)); + let mut nodes = Vec::new(); + visited.take_path(&key, |node| nodes.push(node))?; + for node in nodes { + e = e.context(format!("traversing {}", node.node_key())); + } + return Err(e); + } + } + } + + Ok(None) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::collections::HashSet; + + use async_trait::async_trait; + use buck2_query::query::traversal::ChildVisitor; + use dupe::Dupe; + use gazebo::prelude::VecExt; + + use crate::query::graph::async_bfs::async_bfs_find_path; + use crate::query::graph::node::LabeledNode; + use crate::query::graph::node::NodeKey; + use crate::query::graph::successors::AsyncChildVisitor; + use crate::query::traversal::AsyncNodeLookup; + + #[derive(Copy, Clone, Dupe, derive_more::Display, Debug, Eq, PartialEq, Hash)] + #[display(fmt = "{:?}", "self")] + struct TestNodeKey(u32); + #[derive(Copy, Clone, Dupe, Debug, Eq, PartialEq)] + struct TestNode(TestNodeKey); + + impl NodeKey for TestNodeKey {} + + impl LabeledNode for TestNode { + type Key = TestNodeKey; + + fn node_key(&self) -> &Self::Key { + &self.0 + } + } + + #[derive(Default)] + struct TestGraph { + successors: HashMap>, + errors: HashSet, + } + + impl TestGraph { + fn add_edge(&mut self, from: u32, to: u32) { + self.successors.entry(from).or_default().push(to); + } + + fn add_error(&mut self, node: u32) { + self.errors.insert(node); + } + } + + impl TestGraph { + async fn bfs_find_path( + &self, + roots: impl IntoIterator, + target: u32, + ) -> anyhow::Result>> { + let roots: Vec = roots + .into_iter() + .map(|n| TestNode(TestNodeKey(n))) + .collect(); + let path = async_bfs_find_path(&roots, self, self, |n| { + if n.0 == target { + Some(TestNode(*n)) + } else { + None + } + }) + .await?; + Ok(path.map(|path| path.into_map(|n| n.0.0))) + } + } + + impl AsyncChildVisitor for TestGraph { + async fn for_each_child( + &self, + node: &TestNode, + mut children: impl ChildVisitor, + ) -> anyhow::Result<()> { + for succ in self.successors.get(&node.0.0).unwrap_or(&Vec::new()) { + children.visit(&TestNodeKey(*succ))?; + } + Ok(()) + } + } + + #[async_trait] + impl AsyncNodeLookup for TestGraph { + async fn get(&self, label: &TestNodeKey) -> anyhow::Result { + if self.errors.contains(&label.0) { + return Err(anyhow::anyhow!("my error")); + } + Ok(TestNode(*label)) + } + } + + struct SuccessorsPlus1; + + impl AsyncChildVisitor for SuccessorsPlus1 { + async fn for_each_child( + &self, + node: &TestNode, + mut children: impl ChildVisitor, + ) -> anyhow::Result<()> { + children.visit(&TestNodeKey(node.0.0 + 1))?; + Ok(()) + } + } + + struct TestLookupImpl; + + #[async_trait] + impl AsyncNodeLookup for TestLookupImpl { + async fn get(&self, label: &TestNodeKey) -> anyhow::Result { + Ok(TestNode(*label)) + } + } + + #[tokio::test] + async fn test_async_bfs_find_path() { + let mut g = TestGraph::default(); + g.add_edge(0, 1); + g.add_edge(1, 2); + g.add_edge(2, 3); + g.add_edge(3, 4); + g.add_edge(4, 5); + + let path = g.bfs_find_path([0], 0).await.unwrap(); + assert_eq!(Some(vec![0]), path); + + let path = g.bfs_find_path([0], 1).await.unwrap(); + assert_eq!(Some(vec![0, 1]), path); + + let path = g.bfs_find_path([0], 2).await.unwrap(); + assert_eq!(Some(vec![0, 1, 2]), path); + + let path = g.bfs_find_path([0], 3).await.unwrap(); + assert_eq!(Some(vec![0, 1, 2, 3]), path); + } + + #[tokio::test] + async fn test_async_bfs_find_path_branch() { + let mut g = TestGraph::default(); + g.add_edge(0, 1); + g.add_edge(0, 3); + g.add_edge(1, 2); + g.add_edge(2, 3); + g.add_edge(3, 4); + + let path = g.bfs_find_path([0], 0).await.unwrap(); + assert_eq!(Some(vec![0]), path); + + let path = g.bfs_find_path([0], 1).await.unwrap(); + assert_eq!(Some(vec![0, 1]), path); + + let path = g.bfs_find_path([0], 2).await.unwrap(); + assert_eq!(Some(vec![0, 1, 2]), path); + + let path = g.bfs_find_path([0], 3).await.unwrap(); + assert_eq!(Some(vec![0, 3]), path); + + let path = g.bfs_find_path([0], 4).await.unwrap(); + assert_eq!(Some(vec![0, 3, 4]), path); + } + + #[tokio::test] + async fn test_async_bfs_find_path_error() { + let mut g = TestGraph::default(); + g.add_edge(0, 1); + g.add_edge(1, 2); + g.add_edge(2, 3); + g.add_error(3); + + let err = g.bfs_find_path([0], 9).await.unwrap_err(); + + let errors: Vec = err.chain().map(|e| e.to_string()).collect(); + assert_eq!( + vec![ + "traversing TestNodeKey(0)", + "traversing TestNodeKey(1)", + "traversing TestNodeKey(2)", + "traversing TestNodeKey(3)", + "my error" + ], + errors + ); + } + + #[tokio::test] + async fn test_async_bfs_find_path_multiple_starts() { + let mut g = TestGraph::default(); + g.add_edge(0, 1); + g.add_edge(1, 2); + g.add_edge(2, 3); + g.add_edge(3, 4); + g.add_edge(10, 11); + g.add_edge(11, 12); + + let path = g.bfs_find_path([0, 10], 12).await.unwrap(); + assert_eq!(Some(vec![10, 11, 12]), path); + } + + #[tokio::test] + async fn test_async_bfs_find_path_no_path() { + let mut g = TestGraph::default(); + g.add_edge(0, 1); + g.add_edge(1, 2); + g.add_edge(1, 3); + g.add_edge(1, 4); + g.add_edge(2, 3); + g.add_edge(3, 4); + + let path = g.bfs_find_path([0], 10).await.unwrap(); + assert_eq!(None, path); + } +} diff --git a/app/buck2_query/src/query/graph/bfs.rs b/app/buck2_query/src/query/graph/bfs.rs new file mode 100644 index 0000000000000..be244dee63ac9 --- /dev/null +++ b/app/buck2_query/src/query/graph/bfs.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +//! Generic BFS implementation. + +use std::collections::VecDeque; +use std::hash::Hash; + +use starlark_map::unordered_set; +use starlark_map::unordered_set::UnorderedSet; +use starlark_map::Hashed; + +use crate::query::graph::successors::GraphSuccessors; + +pub fn bfs_preorder( + roots: impl IntoIterator, + successors: impl GraphSuccessors, + mut visit: impl FnMut(N), +) { + let mut visited: UnorderedSet = UnorderedSet::new(); + let mut work: VecDeque = VecDeque::new(); + for root in roots { + let root = Hashed::new(root); + match visited.raw_entry_mut().from_entry_hashed(root.as_ref()) { + unordered_set::RawEntryMut::Occupied(_) => {} + unordered_set::RawEntryMut::Vacant(entry) => { + entry.insert_hashed(root.clone()); + work.push_back(root.into_key()); + } + } + } + + while let Some(curr) = work.pop_front() { + successors.for_each_successor(&curr, |succ| { + let succ = Hashed::new(succ); + match visited.raw_entry_mut().from_entry_hashed(succ) { + unordered_set::RawEntryMut::Occupied(_) => {} + unordered_set::RawEntryMut::Vacant(entry) => { + entry.insert_hashed(succ.cloned()); + work.push_back(succ.into_key().clone()); + } + } + }); + visit(curr); + } +} + +#[cfg(test)] +mod tests { + use crate::query::graph::bfs::bfs_preorder; + use crate::query::graph::successors::GraphSuccessors; + + #[test] + fn test_bfs_preorder() { + struct SuccessorsImpl; + + impl GraphSuccessors for SuccessorsImpl { + fn for_each_successor(&self, node: &u32, mut cb: impl FnMut(&u32)) { + for node in [node + 3, node + 5] { + if node <= 10 { + cb(&node); + } + } + } + } + + let mut visited = Vec::new(); + bfs_preorder([0], SuccessorsImpl, |n| visited.push(n)); + + assert_eq!(vec![0, 3, 5, 6, 8, 10, 9], visited); + } +} diff --git a/app/buck2_query/src/query/graph/dfs.rs b/app/buck2_query/src/query/graph/dfs.rs new file mode 100644 index 0000000000000..27235f5eeebeb --- /dev/null +++ b/app/buck2_query/src/query/graph/dfs.rs @@ -0,0 +1,127 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +//! Generic DFS implementation. + +use std::hash::Hash; + +use dupe::Dupe; +use starlark_map::unordered_set::UnorderedSet; + +use crate::query::graph::successors::GraphSuccessors; +use crate::query::graph::vec_as_set::VecAsSet; +use crate::query::graph::visited::VisitedNodes; + +pub fn dfs_postorder( + roots: impl IntoIterator, + successors: impl GraphSuccessors, + visit: impl FnMut(N) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + dfs_postorder_impl::<_, UnorderedSet>(roots, successors, visit) +} + +pub(crate) fn dfs_postorder_impl>( + roots: impl IntoIterator, + successors: impl GraphSuccessors, + mut visit: impl FnMut(N) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + // This implementation simply performs a dfs. We maintain a work stack here. + // When visiting a node, we first add an item to the work stack to call + // post_visit for that node, and then add items to visit all the + // children. While a work item for a child will not be added if it has + // already been visited, if there's an item in the stack for that child + // it will still be added. When popping the visit, if the node had been + // visited, it's ignored. This ensures that a node's children are all + // visited before we do PostVisit for that node. + enum WorkItem { + PostVisit(N), + Visit(H, N), + } + + let mut visited: V = V::default(); + let mut work: Vec> = roots + .into_iter() + .map(|t| WorkItem::Visit(V::hash(&t), t)) + .collect(); + + while let Some(curr) = work.pop() { + match curr { + WorkItem::Visit(hash, target) => { + if !visited.insert_clone(hash, &target) { + continue; + } + + work.push(WorkItem::PostVisit(target.dupe())); + + successors.for_each_successor(&target, |succ| { + let hash = V::hash(succ); + if !visited.contains(hash, succ) { + work.push(WorkItem::Visit(hash, succ.dupe())); + } + }); + } + WorkItem::PostVisit(target) => { + visit(target)?; + } + } + } + + Ok(()) +} + +pub(crate) fn dfs_preorder( + roots: impl IntoIterator, + successors: impl GraphSuccessors, + mut visit: impl FnMut(u32), +) { + let mut visited = VecAsSet::default(); + let mut work = Vec::new(); + + for root in roots { + work.push(root); + + while let Some(node) = work.pop() { + if !visited.insert(node) { + continue; + } + visit(node); + + let work_len = work.len(); + successors.for_each_successor(&node, |succ| { + work.push(*succ); + }); + work[work_len..].reverse(); + } + } +} + +#[cfg(test)] +mod tests { + use crate::query::graph::dfs::dfs_preorder; + use crate::query::graph::successors::GraphSuccessors; + + #[test] + fn test() { + struct SuccessorImpl; + + impl GraphSuccessors for SuccessorImpl { + fn for_each_successor(&self, node: &u32, mut cb: impl FnMut(&u32)) { + for succ in [node + 2, node + 3] { + if succ <= 10 { + cb(&succ); + } + } + } + } + + let mut visited = Vec::new(); + dfs_preorder([0, 1], SuccessorImpl, |n| visited.push(n)); + assert_eq!(vec![0, 2, 4, 6, 8, 10, 9, 7, 5, 3, 1], visited); + } +} diff --git a/app/buck2_query/src/query/graph/graph.rs b/app/buck2_query/src/query/graph/graph.rs new file mode 100644 index 0000000000000..4ce2aa596116e --- /dev/null +++ b/app/buck2_query/src/query/graph/graph.rs @@ -0,0 +1,492 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::collections::VecDeque; + +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use starlark_map::unordered_map; +use starlark_map::unordered_map::UnorderedMap; +use starlark_map::Hashed; + +use crate::query::graph::dfs::dfs_postorder_impl; +use crate::query::graph::dfs::dfs_preorder; +use crate::query::graph::node::LabeledNode; +use crate::query::graph::successors::AsyncChildVisitor; +use crate::query::graph::successors::GraphSuccessors; +use crate::query::graph::vec_as_map::VecAsMap; +use crate::query::graph::vec_as_set::VecAsSet; +use crate::query::traversal::AsyncNodeLookup; + +#[derive(Clone)] +struct GraphNode { + node: N, + children: Vec, +} + +/// Graph with all nodes and edges resolved and represented as integers. +/// +/// This is fast to traverse. +#[derive(Clone)] +pub(crate) struct Graph { + nodes: Vec>, + node_to_index: UnorderedMap, +} + +impl Graph { + pub(crate) fn get(&self, node: &N::Key) -> Option<&N> { + self.node_to_index + .get(node) + .map(|index| &self.nodes[*index as usize].node) + } +} + +struct GraphBuilder { + node_to_index: UnorderedMap, + nodes: VecAsMap>, +} + +impl GraphBuilder { + fn build(self) -> Graph { + assert_eq!(self.nodes.vec.len(), self.node_to_index.len()); + let nodes = self + .nodes + .vec + .into_iter() + .map(|n| n.unwrap()) + .collect::>(); + Graph { + nodes, + node_to_index: self.node_to_index, + } + } + + fn get_or_create_node(&mut self, node: &N::Key) -> u32 { + let node = Hashed::new(node); + let new_index = self.node_to_index.len(); + match self.node_to_index.raw_entry_mut().from_key_hashed(node) { + unordered_map::RawEntryMut::Occupied(e) => *e.get(), + unordered_map::RawEntryMut::Vacant(e) => { + let new_index = new_index.try_into().unwrap(); + e.insert((*node.key()).clone(), new_index); + new_index + } + } + } + + fn insert(&mut self, index: u32, node: N) { + let prev = self.nodes.insert( + index, + GraphNode { + node, + children: Vec::new(), + }, + ); + assert!(prev.is_none()); + } +} + +impl Graph { + /// Build the graph by traversing the nodes in `root` and their children. + /// + /// Resulting graph have node indices assigned non-deterministically. + pub(crate) async fn build( + nodes: &impl AsyncNodeLookup, + root: impl IntoIterator, + successors: impl AsyncChildVisitor, + ) -> anyhow::Result> { + let mut graph = GraphBuilder:: { + nodes: VecAsMap::default(), + node_to_index: UnorderedMap::default(), + }; + + // Map from node to parent node. + let mut visited: VecAsMap> = VecAsMap::default(); + let mut push = |queue: &mut FuturesUnordered<_>, + target_ref: &T::Key, + target_index: u32, + parent: Option| { + if visited.contains_key(target_index) { + return; + } + + visited.insert(target_index, parent); + + let target_ref = target_ref.clone(); + + queue.push(async move { + let result = nodes.get(&target_ref).await; + (target_index, result) + }) + }; + + let mut queue = FuturesUnordered::new(); + + for target in root { + let index = graph.get_or_create_node(&target); + push(&mut queue, &target, index, None); + } + + // TODO(cjhopman): FuturesOrdered/Unordered interacts poorly with tokio cooperative scheduling + // (see https://github.com/rust-lang/futures-rs/issues/2053). Clean this up once a good + // solution there exists. + while let Some((target_index, node)) = tokio::task::unconstrained(queue.next()).await { + let result: anyhow::Result<_> = try { + let node = node?; + + graph.insert(target_index, node.clone()); + + successors + .for_each_child(&node, &mut |child: &T::Key| { + let child_index = graph.get_or_create_node(child); + graph + .nodes + .get_mut(target_index) + .unwrap() + .children + .push(child_index); + push(&mut queue, child, child_index, Some(target_index)); + Ok(()) + }) + .await?; + graph + .nodes + .get_mut(target_index) + .unwrap() + .children + .shrink_to_fit(); + }; + if let Err(mut e) = result { + let mut target = target_index; + while let Some(Some(parent_index)) = visited.get(target) { + match graph.nodes.get(*parent_index) { + None => { + return Err(e.context(format!( + "Node {} has not node assigned (internal error)", + parent_index + ))); + } + Some(parent) => { + e = e.context(format!( + "Error traversing children of {}", + parent.node.node_key() + )); + target = *parent_index; + } + } + } + return Err(e); + } + } + + Ok(graph.build()) + } + + /// Build graph with nodes laid out in stable DFS order. + pub(crate) async fn build_stable_dfs( + nodes: &impl AsyncNodeLookup, + root: impl IntoIterator, + successors: impl AsyncChildVisitor, + ) -> anyhow::Result> { + let root = root.into_iter().collect::>(); + let graph = Self::build(nodes, root.iter().cloned(), successors).await?; + let root = root.into_iter().map(|n| graph.node_to_index[&n]); + let mut old_to_new: VecAsMap = VecAsMap::default(); + + let mut new_index = 0; + graph.dfs_preorder_indices(root, |old_index| { + let prev = old_to_new.insert(old_index, new_index); + assert!(prev.is_none()); + new_index += 1; + }); + + assert_eq!(graph.nodes.len(), new_index as usize); + + Ok(graph.index_remap(|old_index| *old_to_new.get(old_index).unwrap())) + } + + fn dfs_preorder_indices(&self, roots: impl IntoIterator, visitor: impl FnMut(u32)) { + dfs_preorder(roots, GraphSuccessorsImpl { graph: self }, visitor) + } + + /// Remap the indices of the graph. + fn index_remap(self, remap: impl Fn(u32) -> u32) -> Self { + let node_count = self.nodes.len().try_into().unwrap(); + self.index_remap_opt(|i| Some(remap(i)), node_count) + } + + /// Remap the indices of the graph. + /// + /// `remap` function must map populate the range `0..count`, otherwise this function will panic. + fn index_remap_opt(self, remap: impl Fn(u32) -> Option, count: u32) -> Self { + let Graph { + nodes, + mut node_to_index, + } = self; + + let mut new_nodes: VecAsMap> = VecAsMap::default(); + + for (i, mut node) in nodes.into_iter().enumerate() { + let old_id: u32 = i.try_into().unwrap(); + let Some(new_id) = remap(old_id) else { + continue; + }; + assert!(new_id < count); + + node.children.retain_mut(|node| { + if let Some(new_node) = remap(*node) { + *node = new_node; + true + } else { + false + } + }); + let prev = new_nodes.insert(new_id, node); + assert!(prev.is_none()); + } + + node_to_index.retain(|_, index| { + if let Some(new_index) = remap(*index) { + *index = new_index; + true + } else { + false + } + }); + + let new_nodes = new_nodes.vec.into_iter().map(|n| n.unwrap()).collect(); + Graph { + nodes: new_nodes, + node_to_index, + } + } + + /// Reverse the edges. + pub(crate) fn reverse(self) -> Graph { + let Graph { + mut nodes, + node_to_index, + } = self; + let mut new_edges: Vec> = (0..nodes.len()).map(|_| Vec::new()).collect(); + for node in nodes.iter().enumerate() { + for child in &node.1.children { + new_edges[*child as usize].push(node.0 as u32); + } + } + for (node, new_edges) in nodes.iter_mut().zip(new_edges) { + node.children = new_edges; + } + Graph { + nodes, + node_to_index, + } + } + + pub(crate) fn depth_first_postorder_traversal>( + &self, + root: RootIter, + mut visitor: impl FnMut(&T) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { + dfs_postorder_impl::<_, VecAsSet>( + root.into_iter().map(|root| self.node_to_index[&root]), + GraphSuccessorsImpl { graph: self }, + |index| visitor(&self.nodes[index as usize].node), + ) + } + + /// Create a graph from the given roots up to the given max depth. + /// + /// Zero depth means only the roots. + pub(crate) fn take_max_depth( + self, + roots: impl IntoIterator, + max_depth: u32, + ) -> Graph { + // Map from old index to new index. + let mut visited: VecAsMap = VecAsMap::default(); + let mut ids_to_keep = Vec::new(); + let mut edge: VecDeque = VecDeque::new(); + + for root in roots { + let root = self.node_to_index[&root]; + if !visited.contains_key(root) { + let new_index = ids_to_keep.len().try_into().unwrap(); + let prev = visited.insert(root, new_index); + assert!(prev.is_none()); + ids_to_keep.push(root); + edge.push_back(root); + } + } + + for _ in 0..max_depth { + for _ in 0..edge.len() { + let node = edge.pop_front().unwrap(); + for &succ in &self.nodes[node as usize].children { + if !visited.contains_key(succ) { + let new_index = ids_to_keep.len().try_into().unwrap(); + let prev = visited.insert(succ, new_index); + assert!(prev.is_none()); + ids_to_keep.push(succ); + edge.push_back(succ); + } + } + } + } + + if self.nodes.len() == ids_to_keep.len() { + // We visited everything. Skip expensive remap. + return self; + } + + self.index_remap_opt( + |i| visited.get(i).copied(), + ids_to_keep.len().try_into().unwrap(), + ) + } +} + +struct GraphSuccessorsImpl<'a, N: LabeledNode> { + graph: &'a Graph, +} + +impl<'a, N: LabeledNode> GraphSuccessors for GraphSuccessorsImpl<'a, N> { + fn for_each_successor(&self, node: &u32, mut cb: impl FnMut(&u32)) { + for child in &self.graph.nodes[*node as usize].children { + cb(child); + } + } +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use buck2_query::query::traversal::ChildVisitor; + use dupe::Dupe; + + use crate::query::graph::bfs::bfs_preorder; + use crate::query::graph::graph::Graph; + use crate::query::graph::graph::GraphSuccessorsImpl; + use crate::query::graph::node::LabeledNode; + use crate::query::graph::node::NodeKey; + use crate::query::graph::successors::AsyncChildVisitor; + use crate::query::traversal::AsyncNodeLookup; + + #[derive(Clone, Copy, Dupe, Eq, PartialEq, Hash, derive_more::Display, Debug)] + #[display(fmt = "{}", _0)] + struct Ref(u32); + + #[derive(Clone, Dupe)] + struct Node(Ref); + + impl NodeKey for Ref {} + + impl LabeledNode for Node { + type Key = Ref; + + fn node_key(&self) -> &Self::Key { + &self.0 + } + } + + async fn build_graph(start: &[u32], edges: &[(u32, u32)]) -> Graph { + struct Lookup; + + #[async_trait] + impl AsyncNodeLookup for Lookup { + async fn get(&self, label: &Ref) -> anyhow::Result { + Ok(Node(label.dupe())) + } + } + + struct Successors { + edges: Vec<(u32, u32)>, + } + + impl AsyncChildVisitor for Successors { + async fn for_each_child( + &self, + node: &Node, + mut children: impl ChildVisitor, + ) -> anyhow::Result<()> { + for (from, to) in &self.edges { + if node.0.0 == *from { + children.visit(&Ref(*to))?; + } + } + Ok(()) + } + } + + Graph::build( + &Lookup, + start.iter().copied().map(Ref), + Successors { + edges: edges.to_vec(), + }, + ) + .await + .unwrap() + } + + #[tokio::test] + async fn test_build_then_dfs_postorder() { + let graph = build_graph(&[10], &[(10, 20), (10, 30)]).await; + + let mut visited = Vec::new(); + graph + .depth_first_postorder_traversal([Ref(10)], |node| { + visited.push(node.0.0); + Ok(()) + }) + .unwrap(); + + // TODO(nga): should be `[30, 10, 20]`. + assert_eq!(vec![30, 20, 10], visited); + } + + fn bfs(graph: &Graph, start: &[u32]) -> Vec { + let mut visited = Vec::new(); + bfs_preorder( + start.iter().map(|i| graph.node_to_index[&Ref(*i)]), + GraphSuccessorsImpl { graph }, + |node| { + visited.push(graph.nodes[node as usize].node.0.0); + }, + ); + visited + } + + #[tokio::test] + async fn test_take_max_depth() { + let graph = build_graph(&[10, 30], &[(10, 20), (10, 30), (20, 30), (30, 40)]).await; + + let graph0 = graph.clone().take_max_depth([Ref(10)], 0); + assert_eq!(vec![10], bfs(&graph0, &[10])); + + let graph1 = graph.clone().take_max_depth([Ref(10)], 1); + assert_eq!(vec![10, 20, 30], bfs(&graph1, &[10])); + + let graph2 = graph.clone().take_max_depth([Ref(10)], 2); + assert_eq!(vec![10, 20, 30, 40], bfs(&graph2, &[10])); + + let graph3 = graph.clone().take_max_depth([Ref(10)], 3); + assert_eq!(vec![10, 20, 30, 40], bfs(&graph3, &[10])); + + let graph4 = graph.clone().take_max_depth([Ref(10)], 4); + assert_eq!(vec![10, 20, 30, 40], bfs(&graph4, &[10])); + + let graph_2_0 = graph.clone().take_max_depth([Ref(10), Ref(30)], 0); + assert_eq!(vec![10, 30], bfs(&graph_2_0, &[10, 30])); + + let graph_2_1 = graph.clone().take_max_depth([Ref(10), Ref(30)], 1); + assert_eq!(vec![10, 30, 20, 40], bfs(&graph_2_1, &[10, 30])); + + graph.take_max_depth([], 100); + } +} diff --git a/app/buck2_query/src/query/graph/mod.rs b/app/buck2_query/src/query/graph/mod.rs new file mode 100644 index 0000000000000..f3e328b6dd4f9 --- /dev/null +++ b/app/buck2_query/src/query/graph/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +pub(crate) mod async_bfs; +pub mod bfs; +pub mod dfs; +#[allow(clippy::module_inception)] +pub(crate) mod graph; +pub mod node; +pub mod successors; +pub(crate) mod vec_as_map; +pub(crate) mod vec_as_set; +pub(crate) mod visited; diff --git a/app/buck2_query/src/query/graph/node.rs b/app/buck2_query/src/query/graph/node.rs new file mode 100644 index 0000000000000..1075a93a68960 --- /dev/null +++ b/app/buck2_query/src/query/graph/node.rs @@ -0,0 +1,27 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::fmt::Debug; +use std::fmt::Display; +use std::hash::Hash; + +use dupe::Dupe; +use starlark_map::Hashed; + +pub trait NodeKey: Clone + Hash + PartialEq + Eq + Debug + Display + Send + Sync + 'static {} + +pub trait LabeledNode: Dupe + Send + Sync { + type Key: NodeKey; + + fn node_key(&self) -> &Self::Key; + + fn hashed_node_key(&self) -> Hashed<&Self::Key> { + Hashed::new(self.node_key()) + } +} diff --git a/app/buck2_query/src/query/graph/successors.rs b/app/buck2_query/src/query/graph/successors.rs new file mode 100644 index 0000000000000..de9d5dbb1f341 --- /dev/null +++ b/app/buck2_query/src/query/graph/successors.rs @@ -0,0 +1,35 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::future::Future; + +use buck2_query::query::traversal::ChildVisitor; + +use crate::query::graph::node::LabeledNode; + +/// Function to return the successors of a node. +pub trait GraphSuccessors { + fn for_each_successor(&self, node: &N, cb: impl FnMut(&N)); +} + +pub trait AsyncChildVisitor: Send + Sync { + fn for_each_child( + &self, + node: &N, + children: impl ChildVisitor, + ) -> impl Future> + Send; +} + +impl<'a, N: LabeledNode, A: AsyncChildVisitor + ?Sized + Send + Sync> AsyncChildVisitor + for &'a A +{ + async fn for_each_child(&self, node: &N, children: impl ChildVisitor) -> anyhow::Result<()> { + (**self).for_each_child(node, children).await + } +} diff --git a/app/buck2_query/src/query/graph/vec_as_map.rs b/app/buck2_query/src/query/graph/vec_as_map.rs new file mode 100644 index 0000000000000..022b5908d2ea3 --- /dev/null +++ b/app/buck2_query/src/query/graph/vec_as_map.rs @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +#![allow(dead_code)] // Used later in the stack. + +use std::fmt; +use std::fmt::Debug; +use std::mem; + +/// Map `u32` to `T`. +pub(crate) struct VecAsMap { + pub(crate) vec: Vec>, +} + +impl Debug for VecAsMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries(self.entries()).finish() + } +} + +impl Default for VecAsMap { + fn default() -> Self { + VecAsMap { vec: Vec::new() } + } +} + +impl VecAsMap { + pub(crate) fn get(&self, index: u32) -> Option<&T> { + self.vec.get(index as usize).and_then(|e| e.as_ref()) + } + + pub(crate) fn get_mut(&mut self, index: u32) -> Option<&mut T> { + self.vec.get_mut(index as usize).and_then(|e| e.as_mut()) + } + + pub(crate) fn contains_key(&self, index: u32) -> bool { + self.get(index).is_some() + } + + fn entries(&self) -> impl Iterator { + self.vec + .iter() + .enumerate() + .filter_map(|(i, e)| e.as_ref().map(|e| (i as u32, e))) + } + + pub(crate) fn keys(&self) -> impl Iterator + '_ { + self.entries().map(|(k, _)| k) + } + + pub(crate) fn insert(&mut self, index: u32, value: T) -> Option { + if self.vec.len() <= index as usize { + self.vec.resize_with(index as usize + 1, || None); + } + mem::replace(&mut self.vec[index as usize], Some(value)) + } +} diff --git a/app/buck2_query/src/query/graph/vec_as_set.rs b/app/buck2_query/src/query/graph/vec_as_set.rs new file mode 100644 index 0000000000000..ecbba56f3d7d2 --- /dev/null +++ b/app/buck2_query/src/query/graph/vec_as_set.rs @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +#![allow(dead_code)] // Used later in the stack. + +use std::fmt; +use std::fmt::Debug; + +use crate::query::graph::vec_as_map::VecAsMap; + +#[derive(Default)] +pub(crate) struct VecAsSet { + // Can use bitset here. + vec: VecAsMap<()>, +} + +impl Debug for VecAsSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_set().entries(self.vec.keys()).finish() + } +} + +impl VecAsSet { + pub(crate) fn contains(&self, index: u32) -> bool { + self.vec.contains_key(index) + } + + /// Return true if the index was not already present. + pub(crate) fn insert(&mut self, index: u32) -> bool { + self.vec.insert(index, ()).is_none() + } +} diff --git a/app/buck2_query/src/query/graph/visited.rs b/app/buck2_query/src/query/graph/visited.rs new file mode 100644 index 0000000000000..6148c4899919d --- /dev/null +++ b/app/buck2_query/src/query/graph/visited.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +#![allow(dead_code)] // Used later in the stack. + +use std::hash::Hash; + +use starlark_map::unordered_set; +use starlark_map::unordered_set::UnorderedSet; +use starlark_map::Hashed; +use starlark_map::StarlarkHashValue; + +use crate::query::graph::vec_as_set::VecAsSet; + +/// A set to store visited nodes. +pub(crate) trait VisitedNodes: Default { + type Hash: Copy; + + fn hash(node: &T) -> Self::Hash; + + fn contains(&self, hash: Self::Hash, node: &T) -> bool; + fn insert_clone(&mut self, hash: Self::Hash, node: &T) -> bool + where + T: Clone; +} + +impl VisitedNodes for VecAsSet { + type Hash = (); + + fn hash(_node: &u32) -> Self::Hash {} + + fn contains(&self, _hash: (), node: &u32) -> bool { + self.contains(*node) + } + fn insert_clone(&mut self, _hash: (), node: &u32) -> bool { + self.insert(*node) + } +} + +impl VisitedNodes for UnorderedSet { + type Hash = StarlarkHashValue; + + fn hash(node: &T) -> Self::Hash { + StarlarkHashValue::new(node) + } + + fn contains(&self, hash: Self::Hash, node: &T) -> bool { + self.contains_hashed(Hashed::new_unchecked(hash, node)) + } + + fn insert_clone(&mut self, hash: Self::Hash, node: &T) -> bool + where + T: Clone, + { + match self + .raw_entry_mut() + .from_entry_hashed(Hashed::new_unchecked(hash, node)) + { + unordered_set::RawEntryMut::Occupied(_) => false, + unordered_set::RawEntryMut::Vacant(e) => { + e.insert_hashed(Hashed::new_unchecked(hash, node.clone())); + true + } + } + } +} diff --git a/app/buck2_query/src/query/mod.rs b/app/buck2_query/src/query/mod.rs index 7954a7c8a1c80..68ff672d82b34 100644 --- a/app/buck2_query/src/query/mod.rs +++ b/app/buck2_query/src/query/mod.rs @@ -9,6 +9,6 @@ pub mod buck_types; pub mod environment; -pub(crate) mod futures_queue_generic; +pub mod graph; pub mod syntax; pub mod traversal; diff --git a/app/buck2_query/src/query/syntax/simple/eval/file_set.rs b/app/buck2_query/src/query/syntax/simple/eval/file_set.rs index 188e1e251e7bd..e54e7a04bdeb0 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/file_set.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/file_set.rs @@ -76,7 +76,7 @@ impl FileSet { pub fn owner( &self, - _env: &dyn QueryEnvironment, + _env: &impl QueryEnvironment, ) -> anyhow::Result> { Err(anyhow::anyhow!(QueryError::FunctionUnimplemented( "owner()" diff --git a/app/buck2_query/src/query/syntax/simple/eval/label_indexed.rs b/app/buck2_query/src/query/syntax/simple/eval/label_indexed.rs index fc28640eb843e..c1d9ecf0b6673 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/label_indexed.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/label_indexed.rs @@ -15,52 +15,53 @@ use dupe::Dupe; use starlark_map::ordered_set::OrderedSet; use starlark_map::small_set; use starlark_map::Equivalent; +use starlark_map::Hashed; -use crate::query::environment::LabeledNode; +use crate::query::graph::node::LabeledNode; #[derive(Debug, Clone, Dupe, Allocative)] pub struct LabelIndexed(pub T); impl PartialEq for LabelIndexed { fn eq(&self, other: &Self) -> bool { - self.0.node_ref() == other.0.node_ref() + self.0.node_key() == other.0.node_key() } } impl Eq for LabelIndexed {} impl Hash for LabelIndexed { fn hash(&self, state: &mut H) { - self.0.node_ref().hash(state) + self.0.hashed_node_key().hash().hash(state) } } impl Ord for LabelIndexed where - T::NodeRef: Ord, + T::Key: Ord, { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.0.node_ref().cmp(other.0.node_ref()) + self.0.node_key().cmp(other.0.node_key()) } } impl PartialOrd for LabelIndexed where - T::NodeRef: PartialOrd, + T::Key: PartialOrd, { fn partial_cmp(&self, other: &Self) -> Option { - self.0.node_ref().partial_cmp(other.0.node_ref()) + self.0.node_key().partial_cmp(other.0.node_key()) } } -struct LabelIndexer<'a, T: LabeledNode>(&'a T::NodeRef); +struct LabelIndexer<'a, T: LabeledNode>(Hashed<&'a T::Key>); impl<'a, T: LabeledNode> Equivalent> for LabelIndexer<'a, T> { fn equivalent(&self, key: &LabelIndexed) -> bool { - self.0.eq(key.0.node_ref()) + *self.0.key() == key.0.node_key() } } impl<'a, T: LabeledNode> Hash for LabelIndexer<'a, T> { fn hash(&self, state: &mut H) { - self.0.hash(state) + self.0.hash().hash(state) } } @@ -88,12 +89,16 @@ impl LabelIndexedSet { self.nodes.len() } - pub fn get(&self, value: &T::NodeRef) -> Option<&T> { - self.nodes.get(&LabelIndexer(value)).map(|e| &e.0) + pub fn get(&self, value: &T::Key) -> Option<&T> { + self.nodes + .get(&LabelIndexer(Hashed::new(value))) + .map(|e| &e.0) } - pub fn take(&mut self, value: &T::NodeRef) -> Option { - self.nodes.take(&LabelIndexer(value)).map(|e| e.0) + pub fn take(&mut self, value: &T::Key) -> Option { + self.nodes + .take(&LabelIndexer(Hashed::new(value))) + .map(|e| e.0) } pub fn iter(&self) -> Iter { @@ -111,16 +116,20 @@ impl LabelIndexedSet { self.nodes.insert(LabelIndexed(value)) } - pub fn contains(&self, value: &T::NodeRef) -> bool { - self.nodes.contains(&LabelIndexer(value)) + pub fn insert_unique_unchecked(&mut self, value: T) { + self.nodes.insert_unique_unchecked(LabelIndexed(value)); + } + + pub fn contains(&self, value: &T::Key) -> bool { + self.nodes.contains(&LabelIndexer(Hashed::new(value))) } pub fn get_index(&self, index: usize) -> Option<&T> { self.nodes.get_index(index).map(|e| &e.0) } - pub fn get_index_of(&self, value: &T::NodeRef) -> Option { - self.nodes.get_index_of(&LabelIndexer(value)) + pub fn get_index_of(&self, value: &T::Key) -> Option { + self.nodes.get_index_of(&LabelIndexer(Hashed::new(value))) } pub fn last(&self) -> Option<&T> { diff --git a/app/buck2_query/src/query/syntax/simple/eval/multi_query.rs b/app/buck2_query/src/query/syntax/simple/eval/multi_query.rs index 8b986664e9102..1a0cac875e0ac 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/multi_query.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/multi_query.rs @@ -8,10 +8,7 @@ */ //! Implementation of the cli and query_* attr query language. -use buck2_query_parser::placeholder::QUERY_PERCENT_S_PLACEHOLDER; -use futures::stream::FuturesOrdered; -use futures::Future; -use futures::StreamExt; + use indexmap::IndexMap; use crate::query::environment::QueryTarget; @@ -51,30 +48,3 @@ impl MultiQueryResult { Ok(results) } } - -pub async fn process_multi_query>( - query: &str, - query_args: &[A], - func: F, -) -> MultiQueryResult -where - T: QueryTarget, - Fut: Future>)>, - F: Fn(String, String) -> Fut, -{ - let mut queue: FuturesOrdered<_> = query_args - .iter() - .map(|input| { - let input = input.as_ref(); - let query = query.replace(QUERY_PERCENT_S_PLACEHOLDER, input); - let input = input.to_owned(); - func(input, query) - }) - .collect(); - - let mut results = IndexMap::new(); - while let Some((query, result)) = queue.next().await { - results.insert(query, result); - } - MultiQueryResult(results) -} diff --git a/app/buck2_query/src/query/syntax/simple/eval/set.rs b/app/buck2_query/src/query/syntax/simple/eval/set.rs index 8055a97321ca2..02b3caf4f3d79 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/set.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/set.rs @@ -11,16 +11,13 @@ use std::fmt; use std::fmt::Display; use allocative::Allocative; -use buck2_query::query::environment::LabeledNode; use display_container::fmt_container; use dupe::IterDupedExt; use fancy_regex::Regex; use fancy_regex::RegexBuilder; use indexmap::IndexSet; -use crate::query::environment::NodeLabel; use crate::query::environment::QueryTarget; -use crate::query::syntax::simple::eval::error::QueryError; use crate::query::syntax::simple::eval::file_set::FileNode; use crate::query::syntax::simple::eval::file_set::FileSet; use crate::query::syntax::simple::eval::label_indexed; @@ -48,6 +45,10 @@ impl TargetSet { self.targets.insert(value) } + pub fn insert_unique_unchecked(&mut self, value: T) { + self.targets.insert_unique_unchecked(value) + } + pub fn is_empty(&self) -> bool { self.targets.len() == 0 } @@ -56,11 +57,14 @@ impl TargetSet { self.targets.len() } - fn filter anyhow::Result>(&self, filter: F) -> anyhow::Result> { + pub(crate) fn filter anyhow::Result>( + &self, + filter: F, + ) -> anyhow::Result> { let mut targets = LabelIndexedSet::new(); for target in self.targets.iter() { if filter(target)? { - targets.insert(target.dupe()); + targets.insert_unique_unchecked(target.dupe()); } } Ok(Self { targets }) @@ -85,14 +89,6 @@ impl TargetSet { Ok(FileSet::new(files)) } - // TODO(cjhopman): Does this even make sense? - // TODO(cjhopman): I think this needs a heap to allocate values - pub fn labels(&self, _attr: &str) -> anyhow::Result<()> { - Err(anyhow::anyhow!(QueryError::FunctionUnimplemented( - "labels()" - ))) - } - pub fn union(&self, right: &TargetSet) -> TargetSet { let mut targets = LabelIndexedSet::new(); for target in self.targets.iter() { @@ -104,8 +100,8 @@ impl TargetSet { Self { targets } } - pub fn iter_names(&self) -> impl Iterator + Clone { - self.targets.iter().map(|e| e.node_ref()) + pub fn iter_names(&self) -> impl Iterator + Clone { + self.targets.iter().map(|e| e.node_key()) } pub fn iter(&self) -> Iter { @@ -117,11 +113,11 @@ impl TargetSet { self.targets.into_iter() } - pub fn contains(&self, item: &T::NodeRef) -> bool { + pub fn contains(&self, item: &T::Key) -> bool { self.targets.contains(item) } - pub fn get(&self, item: &T::NodeRef) -> Option<&T> { + pub fn get(&self, item: &T::Key) -> Option<&T> { self.targets.get(item) } @@ -129,7 +125,7 @@ impl TargetSet { self.targets.get_index(index) } - pub fn get_index_of(&self, item: &T::NodeRef) -> Option { + pub fn get_index_of(&self, item: &T::Key) -> Option { self.targets.get_index_of(item) } @@ -172,78 +168,63 @@ impl FromIterator for TargetSet { /// This contains additional TargetSet functions implemented via the core /// functions on TargetSet itself. -pub trait TargetSetExt { - type T: QueryTarget; - - fn filter anyhow::Result>( - &self, - filter: F, - ) -> anyhow::Result>; - - fn attrfilter( +impl TargetSet { + pub fn attrfilter( &self, attribute: &str, filter: &dyn Fn(&str) -> anyhow::Result, - ) -> anyhow::Result> { + ) -> anyhow::Result> { self.filter(move |node| { node.map_attr(attribute, |val| match val { None => Ok(false), - Some(v) => Self::T::attr_any_matches(v, &filter), + Some(v) => T::attr_any_matches(v, &filter), }) }) } - fn nattrfilter( + pub(crate) fn nattrfilter( &self, attribute: &str, filter: &dyn Fn(&str) -> anyhow::Result, - ) -> anyhow::Result> { + ) -> anyhow::Result> { self.filter(move |node| { node.map_attr(attribute, |val| match val { None => Ok(false), - Some(v) => Ok(!Self::T::attr_any_matches(v, &filter)?), + Some(v) => Ok(!T::attr_any_matches(v, &filter)?), }) }) } - fn attrregexfilter(&self, attribute: &str, value: &str) -> anyhow::Result> { + pub fn attrregexfilter(&self, attribute: &str, value: &str) -> anyhow::Result> { let regex = Regex::new(value)?; let filter = move |s: &'_ str| -> anyhow::Result { Ok(regex.is_match(s)?) }; self.attrfilter(attribute, &filter) } /// Filter targets by fully qualified name using regex partial match. - fn filter_name(&self, regex: &str) -> anyhow::Result> { + pub fn filter_name(&self, regex: &str) -> anyhow::Result> { let mut re = RegexBuilder::new(regex); re.delegate_dfa_size_limit(100 << 20); let re = re.build()?; - self.filter(|node| Ok(re.is_match(&node.node_ref().label_for_filter())?)) + self.filter(|node| Ok(re.is_match(&node.label_for_filter())?)) } - fn kind(&self, regex: &str) -> anyhow::Result> { + pub fn kind(&self, regex: &str) -> anyhow::Result> { let re = Regex::new(regex)?; self.filter(|node| Ok(re.is_match(&node.rule_type())?)) } - fn intersect(&self, right: &TargetSet) -> anyhow::Result> { - self.filter(|node| Ok(right.contains(node.node_ref()))) + pub fn intersect(&self, right: &TargetSet) -> anyhow::Result> { + self.filter(|node| Ok(right.contains(node.node_key()))) } - fn difference(&self, right: &TargetSet) -> anyhow::Result> { - self.filter(|node| Ok(!right.contains(node.node_ref()))) - } -} - -impl TargetSetExt for TargetSet { - type T = T; - - fn filter anyhow::Result>(&self, filter: F) -> anyhow::Result> { - TargetSet::filter(self, filter) + pub fn difference(&self, right: &TargetSet) -> anyhow::Result> { + self.filter(|node| Ok(!right.contains(node.node_key()))) } } impl Display for TargetSet { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt_container(f, "[", "]", self.targets.iter().map(|t| t.node_ref())) + fmt_container(f, "[", "]", self.targets.iter().map(|t| t.node_key())) } } diff --git a/app/buck2_query/src/query/syntax/simple/eval/tests.rs b/app/buck2_query/src/query/syntax/simple/eval/tests.rs index fe80fd6dba4b1..2eae22f50ef89 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/tests.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/tests.rs @@ -17,38 +17,36 @@ use async_trait::async_trait; use buck2_core::build_file_path::BuildFilePath; use buck2_core::cells::cell_path::CellPath; use buck2_core::configuration::compatibility::MaybeCompatible; -use buck2_query::query::environment::LabeledNode; use buck2_query_parser::parse_expr; use derive_more::Display; use dupe::Dupe; -use serde::Serialize; -use serde::Serializer; -use crate::query::environment::NodeLabel; use crate::query::environment::QueryEnvironment; use crate::query::environment::QueryTarget; +use crate::query::graph::node::LabeledNode; +use crate::query::graph::node::NodeKey; +use crate::query::graph::successors::AsyncChildVisitor; use crate::query::syntax::simple::eval::error::QueryError; use crate::query::syntax::simple::eval::evaluator::QueryEvaluator; use crate::query::syntax::simple::eval::file_set::FileSet; use crate::query::syntax::simple::eval::set::TargetSet; use crate::query::syntax::simple::functions::DefaultQueryFunctionsModule; -use crate::query::traversal::AsyncTraversalDelegate; #[derive(Clone, Hash, PartialEq, Eq, Debug, Display)] struct TargetRef(String); -impl NodeLabel for TargetRef {} +impl NodeKey for TargetRef {} -#[derive(Debug, Display, Serialize)] +#[derive(Debug, Display)] struct TargetAttr(String); #[derive(Debug, Clone, Dupe, Eq, PartialEq)] struct Target {} impl LabeledNode for Target { - type NodeRef = TargetRef; + type Key = TargetRef; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { unimplemented!() } } @@ -68,15 +66,15 @@ impl QueryTarget for Target { unimplemented!() } - fn deps<'a>(&'a self) -> Box + Send + 'a> { + fn deps<'a>(&'a self) -> Box + Send + 'a> { unimplemented!() } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { + fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { unimplemented!() } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { + fn target_deps<'a>(&'a self) -> Box + Send + 'a> { unimplemented!() } @@ -104,22 +102,6 @@ impl QueryTarget for Target { fn map_attr>) -> R>(&self, _key: &str, _func: F) -> R { unimplemented!() } - - fn call_stack(&self) -> Option { - None - } - - fn attr_to_string_alternate(&self, _attr: &Self::Attr<'_>) -> String { - unimplemented!("not needed for tests") - } - - fn attr_serialize( - &self, - _attr: &Self::Attr<'_>, - _serializer: S, - ) -> Result { - unimplemented!("not needed for tests") - } } struct Env; @@ -149,7 +131,8 @@ impl QueryEnvironment for Env { async fn dfs_postorder( &self, _root: &TargetSet, - _delegate: &mut dyn AsyncTraversalDelegate, + _delegate: impl AsyncChildVisitor, + _visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { unimplemented!() } @@ -157,7 +140,8 @@ impl QueryEnvironment for Env { async fn depth_limited_traversal( &self, _root: &TargetSet, - _delegate: &mut dyn AsyncTraversalDelegate, + _delegate: impl AsyncChildVisitor, + _visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, _depth: u32, ) -> anyhow::Result<()> { unimplemented!() diff --git a/app/buck2_query/src/query/syntax/simple/eval/values.rs b/app/buck2_query/src/query/syntax/simple/eval/values.rs index 031241eb2d0c1..d4d3ce229f623 100644 --- a/app/buck2_query/src/query/syntax/simple/eval/values.rs +++ b/app/buck2_query/src/query/syntax/simple/eval/values.rs @@ -52,13 +52,11 @@ impl QueryEvaluationValue { pub fn try_into_targets(self) -> anyhow::Result> { match self { QueryEvaluationValue::TargetSet(targets) => Ok(targets), - v => { - return Err(QueryError::InvalidType { - expected: "targets", - actual: v.variant_name(), - } - .into()); + v => Err(QueryError::InvalidType { + expected: "targets", + actual: v.variant_name(), } + .into()), } } } diff --git a/app/buck2_query/src/query/syntax/simple/functions/mod.rs b/app/buck2_query/src/query/syntax/simple/functions/mod.rs index 1387ad622fd52..f9d016c58e847 100644 --- a/app/buck2_query/src/query/syntax/simple/functions/mod.rs +++ b/app/buck2_query/src/query/syntax/simple/functions/mod.rs @@ -25,7 +25,6 @@ use crate::query::syntax::simple::eval::error::QueryError; use crate::query::syntax::simple::eval::evaluator::QueryEvaluator; use crate::query::syntax::simple::eval::file_set::FileSet; use crate::query::syntax::simple::eval::set::TargetSet; -use crate::query::syntax::simple::eval::set::TargetSetExt; use crate::query::syntax::simple::eval::values::QueryResult; use crate::query::syntax::simple::eval::values::QueryValue; use crate::query::syntax::simple::eval::values::QueryValueSet; @@ -228,6 +227,15 @@ impl DefaultQueryFunctionsModule { Ok(self.implementation.somepath(env, &from, &to).await?.into()) } + /// The `attrfilter(attribute, value, targets)` operator evaluates the given target expression and filters the resulting build targets to those where the specified attribute contains the specified value. + /// In this context, the term attribute refers to an argument in a build rule, such as name, headers, srcs, or deps. + /// + /// - If the attribute is a single value, say `name`, it is compared to the specified value, and the target is returned if they match. + /// - If the attribute is a list, the target is returned if that list contains the specified value. + /// - If the attribute is a dictionary, the target is returned if the value exists in either the keys or the values of the dictionary. + /// + /// For example: + /// `buck2 query "attrfilter(deps, '//foo:bar', '//...')"` returns the build targets in the repository that depend on `//foo:bar`, or more precisely: those build targets that include `//foo:bar` in their deps argument list. async fn attrfilter( &self, attr: String, @@ -332,6 +340,9 @@ impl DefaultQueryFunctionsModule { Ok(self.implementation.inputs(&targets)?.into()) } + /// The `kind(regex, targets)` operator evaluates the specified target expression, `targets`, and returns the targets where the rule type matches the specified `regex`. + /// The specified pattern can be a regular expression. For example, + /// `buck2 query "kind('java.*', deps('//foo:bar'))"` returns the targets that match the rule type `java.*` (`java_library`, `java_binary`, etc.) in the transitive dependencies of `//foo:bar`. async fn kind(&self, regex: String, targets: TargetSet) -> QueryFuncResult { Ok(targets.kind(®ex)?.into()) } @@ -347,6 +358,14 @@ impl DefaultQueryFunctionsModule { self.implementation.labels(&attr, &targets) } + /// The `owner(inputfile)` operator returns the targets that own the specified inputfile. + /// In this context, own means that the target has the specified file as an input. You could consider the `owner()` and `inputs()` operators to be inverses of each other. + /// + /// Example: `buck2 query "owner('examples/1.txt')"` returns the targets that owns the file `examples/1.txt`, which could be a value such as `//examples:one`. + /// + /// It is possible for the specified file to have multiple owners, in which case, owner() returns a set of targets. + /// + /// If no owner for the file is found, owner() outputs the message: `No owner was found for ` async fn owner(&self, env: &Env, files: FileSet) -> QueryFuncResult { Ok(self.implementation.owner(env, &files).await?.into()) } @@ -373,7 +392,7 @@ impl DefaultQueryFunctionsModule { // of a deps functions 3rd parameter expr. When used in that context, the QueryFunctions will be augmented to // have non-erroring implementations. async fn first_order_deps(&self) -> QueryFuncResult { - self.implementation.first_order_deps() + Err(QueryError::NotAvailableInContext("first_order_deps")) } async fn target_deps(&self) -> QueryFuncResult { Err(QueryError::NotAvailableInContext("target_deps")) @@ -381,6 +400,7 @@ impl DefaultQueryFunctionsModule { async fn exec_deps(&self) -> QueryFuncResult { Err(QueryError::NotAvailableInContext("exec_deps")) } + /// Computes the set intersection over the given arguments. /// Can be used with the `^` symbol. This operator is commutative. /// @@ -469,6 +489,26 @@ impl DefaultQueryFunctions { Ok(env.allpaths(from, to).await?) } + /// Find the shortest path from one target set to another. + /// + /// First parameter is downstream (for example, final binary), second is upstream (for example, a library). + /// + /// If there are multiple paths, which one is returned is unspecified. + /// + /// Results are returned in order from up to down. + /// + /// If there's no path, return an empty set. + /// + /// # Example + /// + /// ```text + /// $ buck2 uquery 'somepath(fbcode//buck2:buck2, fbcode//buck2/app/buck2_node:buck2_node)' + /// + /// fbcode//buck2/app/buck2_node:buck2_node + /// fbcode//buck2/app/buck2_analysis:buck2_analysis + /// fbcode//buck2/app/buck2:buck2-bin + /// fbcode//buck2:buck2 + /// ``` pub async fn somepath( &self, env: &Env, @@ -600,19 +640,6 @@ impl DefaultQueryFunctions { env.testsof_with_default_target_platform(targets).await } - // These three functions are intentionally implemented as errors. They are only available within the context - // of a deps functions 3rd parameter expr. When used in that context, the QueryFunctions will be augmented to - // have non-erroring implementations. - pub fn first_order_deps(&self) -> QueryFuncResult { - Err(QueryError::NotAvailableInContext("first_order_deps")) - } - pub fn target_deps(&self) -> QueryFuncResult { - Err(QueryError::NotAvailableInContext("target_deps")) - } - pub fn exec_deps(&self) -> QueryFuncResult { - Err(QueryError::NotAvailableInContext("exec_deps")) - } - pub async fn intersect( &self, env: &Env, diff --git a/app/buck2_query/src/query/traversal.rs b/app/buck2_query/src/query/traversal.rs index bffb971c70a52..a49684d0c1fa1 100644 --- a/app/buck2_query/src/query/traversal.rs +++ b/app/buck2_query/src/query/traversal.rs @@ -11,66 +11,59 @@ use std::collections::HashMap; use std::collections::HashSet; use async_trait::async_trait; +use futures::stream::FuturesOrdered; use futures::StreamExt; +use starlark_map::StarlarkHasherBuilder; -use crate::query::environment::LabeledNode; -use crate::query::futures_queue_generic::FuturesQueue; -use crate::query::syntax::simple::eval::label_indexed::LabelIndexedSet; +use crate::query::graph::graph::Graph; +use crate::query::graph::node::LabeledNode; +use crate::query::graph::successors::AsyncChildVisitor; pub trait ChildVisitor: Send { - fn visit(&mut self, node: T::NodeRef) -> anyhow::Result<()>; + fn visit(&mut self, node: &T::Key) -> anyhow::Result<()>; } impl ChildVisitor for F where - F: FnMut(T::NodeRef) -> anyhow::Result<()>, + F: FnMut(&T::Key) -> anyhow::Result<()>, F: Send, { - fn visit(&mut self, node: T::NodeRef) -> anyhow::Result<()> { + fn visit(&mut self, node: &T::Key) -> anyhow::Result<()> { self(node) } } -/// The TraversalDelegate determines how to traverse the graph (via -/// for_each_child) and then handles doing the actual processing (via -/// visit). Different traversals may call `visit()` at different times (ex. dfs_postorder -/// calls it after all children have been visited) -#[async_trait] -pub trait AsyncTraversalDelegate: Send + Sync { - /// visit is called once for each node. When it is called is traversal-dependent. - fn visit(&mut self, target: T) -> anyhow::Result<()>; - - /// for_each_child should apply the provided function to each child of the node. This may be called multiple times in some traversals. - async fn for_each_child( - &mut self, - target: &T, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()>; -} - pub trait NodeLookup { // TODO(cjhopman): Maybe this should be `&mut self` since we only need the one reference to it. - fn get(&self, label: &T::NodeRef) -> anyhow::Result; + fn get(&self, label: &T::Key) -> anyhow::Result; } #[async_trait] pub trait AsyncNodeLookup: Send + Sync { - async fn get(&self, label: &T::NodeRef) -> anyhow::Result; + async fn get(&self, label: &T::Key) -> anyhow::Result; } -/// Implements a completely unordered graph traversal that visits nodes in a random order. When -/// traversing the graph (potentially) requires work to produce each node (like processing build files) -/// this unordered traversal will parallelize the work efficiently. -pub async fn async_unordered_traversal< - 'a, - T: LabeledNode, - RootIter: IntoIterator, ->( - nodes: &dyn AsyncNodeLookup, - root: RootIter, - delegate: &mut dyn AsyncTraversalDelegate, -) -> anyhow::Result<()> { - async_traversal_common(nodes, root, delegate, None, false).await +/// Node lookup when node key is the same as the node. +pub struct NodeLookupId; + +impl> NodeLookup for NodeLookupId { + fn get(&self, key: &T::Key) -> anyhow::Result { + Ok(key.dupe()) + } +} + +#[async_trait] +impl> AsyncNodeLookup for NodeLookupId { + async fn get(&self, key: &T::Key) -> anyhow::Result { + Ok(key.dupe()) + } +} + +#[async_trait] +impl<'a, T: LabeledNode, A: AsyncNodeLookup> AsyncNodeLookup for &'a A { + async fn get(&self, label: &T::Key) -> anyhow::Result { + (*self).get(label).await + } } /// Implements a depth-first postorder traversal. A node will be visited only after all of its @@ -81,13 +74,13 @@ pub async fn async_unordered_traversal< // TODO(cjhopman): Figure out how to implement this traversal in a way that has good performance // in both cases. pub async fn async_fast_depth_first_postorder_traversal< - 'a, T: LabeledNode, - RootIter: IntoIterator, + RootIter: IntoIterator, >( - nodes: &(dyn NodeLookup + Send + Sync), + nodes: &impl NodeLookup, root: RootIter, - delegate: &mut dyn AsyncTraversalDelegate, + successors: impl AsyncChildVisitor, + mut visit: impl FnMut(T) -> anyhow::Result<()>, ) -> anyhow::Result<()> { // This implementation simply performs a dfs. We maintain a work stack here. // When visiting a node, we first add an item to the work stack to call @@ -97,16 +90,9 @@ pub async fn async_fast_depth_first_postorder_traversal< // it will still be added. When popping the visit, if the node had been // visited, it's ignored. This ensures that a node's children are all // visited before we do PostVisit for that node. - #[derive(Hash, Eq, PartialEq)] enum WorkItem { PostVisit(T), - Visit(T::NodeRef), - } - - #[derive(Default)] - struct State { - visited: HashSet, - work: Vec>, + Visit(T::Key), } // TODO(cjhopman): There's a couple of things that could be improved about this. @@ -114,53 +100,32 @@ pub async fn async_fast_depth_first_postorder_traversal< // couldn't figure out quite a good way to do that in rust. I think it would // mean changing the delegate's for_each_children to return an iterator, // but idk. - impl State { - fn new() -> Self { - Self { - visited: HashSet::new(), - work: Vec::new(), - } - } - - fn push(&mut self, target: T::NodeRef) { - if self.visited.contains(&target) { - return; - } - - self.work.push(WorkItem::Visit(target)); - } - fn pop(&mut self) -> Option> { - self.work.pop() - } - } + let mut visited: HashSet = HashSet::default(); + let mut work: Vec> = root.into_iter().map(|t| WorkItem::Visit(t)).collect(); - let mut state = State::new(); - - for target in root { - state.push(target.clone()); - } - - while let Some(curr) = state.pop() { + while let Some(curr) = work.pop() { match curr { WorkItem::Visit(target) => { - if state.visited.contains(&target) { + if visited.contains(&target) { continue; } let node = nodes.get(&target)?; - state.visited.insert(target); - state.work.push(WorkItem::PostVisit(node.dupe())); - - delegate - .for_each_child(&node, &mut |child| { - state.push(child); + visited.insert(target); + work.push(WorkItem::PostVisit(node.dupe())); + + successors + .for_each_child(&node, &mut |child: &T::Key| { + if !visited.contains(child) { + work.push(WorkItem::Visit(child.clone())); + } Ok(()) }) .await?; } WorkItem::PostVisit(target) => { - delegate.visit(target)?; + visit(target)?; } } } @@ -168,41 +133,35 @@ pub async fn async_fast_depth_first_postorder_traversal< Ok(()) } -async fn async_traversal_common< +pub async fn async_depth_limited_traversal< 'a, - T: LabeledNode, - RootIter: IntoIterator, + T: LabeledNode + 'static, + RootIter: IntoIterator, >( - nodes: &dyn AsyncNodeLookup, + nodes: &impl AsyncNodeLookup, root: RootIter, - delegate: &mut dyn AsyncTraversalDelegate, - // `None` means no max depth. - max_depth: Option, - ordered: bool, + successors: impl AsyncChildVisitor, + mut visit: impl FnMut(T) -> anyhow::Result<()>, + max_depth: u32, ) -> anyhow::Result<()> { - let mut visited = HashMap::new(); - let mut push = |queue: &mut FuturesQueue<_>, - target: T::NodeRef, - parent: Option, - depth: u32| { - if visited.contains_key(&target) { - return; - } - visited.insert(target.clone(), parent); - queue.push(async move { - let result = nodes.get(&target).await; - (target, depth, result) - }) - }; - - let mut queue = if ordered { - FuturesQueue::new_ordered() - } else { - FuturesQueue::new_unordered() - }; + let mut visited: HashMap<_, _, StarlarkHasherBuilder> = HashMap::default(); + let mut push = + |queue: &mut FuturesOrdered<_>, target: &T::Key, parent: Option, depth: u32| { + if visited.contains_key(target) { + return; + } + visited.insert(target.clone(), parent); + let target = target.clone(); + queue.push_back(async move { + let result = nodes.get(&target).await; + (target, depth, result) + }) + }; + + let mut queue = FuturesOrdered::new(); for target in root { - push(&mut queue, target.clone(), None, 0); + push(&mut queue, target, None, 0); } // TODO(cjhopman): FuturesOrdered/Unordered interacts poorly with tokio cooperative scheduling @@ -211,17 +170,17 @@ async fn async_traversal_common< while let Some((target, depth, node)) = tokio::task::unconstrained(queue.next()).await { let result: anyhow::Result<_> = try { let node = node?; - if Some(depth) != max_depth { + if depth != max_depth { let depth = depth + 1; - delegate - .for_each_child(&node, &mut |child| { + successors + .for_each_child(&node, &mut |child: &T::Key| { push(&mut queue, child, Some(target.clone()), depth); Ok(()) }) .await?; } - delegate.visit(node)?; + visit(node)?; }; if let Err(mut e) = result { @@ -237,90 +196,22 @@ async fn async_traversal_common< Ok(()) } -pub async fn async_depth_limited_traversal< - 'a, - T: LabeledNode, - RootIter: IntoIterator, ->( - nodes: &dyn AsyncNodeLookup, - root: RootIter, - delegate: &mut dyn AsyncTraversalDelegate, - max_depth: u32, -) -> anyhow::Result<()> { - async_traversal_common(nodes, root, delegate, Some(max_depth), true).await -} - /// Implements a depth-first postorder traversal. A node will be visited only after all of its /// dependencies have been visited. // TODO(cjhopman): Accept a generic iterator for the roots. We need to iterate over it twice and it's only used with this specific iterator so it was easier to not be generic. pub async fn async_depth_first_postorder_traversal< 'a, T: LabeledNode, - Iter: IntoIterator + Clone, + Iter: IntoIterator + Clone, >( - nodes: &dyn AsyncNodeLookup, + nodes: &impl AsyncNodeLookup, root: Iter, - delegate: &mut dyn AsyncTraversalDelegate, + successors: impl AsyncChildVisitor, + mut visit: impl FnMut(T) -> anyhow::Result<()>, ) -> anyhow::Result<()> { - // We first do an unordered graph traversal to collect all nodes. The unordered traversal efficiently - // uses resources when we need to process build files. - // We don't cache the results of the for_each_child iterators, so that is called multiple times. Potentially it would be more performant to avoid that if an expensive filter/operation is involved. - struct UnorderedDelegate<'a, T: LabeledNode> { - delegate: &'a mut dyn AsyncTraversalDelegate, - nodes: LabelIndexedSet, - } - - #[async_trait] - impl AsyncTraversalDelegate for UnorderedDelegate<'_, T> { - /// visit is called once for each node. When it is called is traversal-dependent. - fn visit(&mut self, target: T) -> anyhow::Result<()> { - self.nodes.insert(target); - Ok(()) - } - - /// for_each_child should apply the provided function to each child of the node. This may be called multiple times in some traversals. - async fn for_each_child( - &mut self, - target: &T, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - self.delegate.for_each_child(target, func).await - } - } - - let mut unordered_delegate = UnorderedDelegate { - delegate, - nodes: LabelIndexedSet::new(), - }; - - async_unordered_traversal(nodes, root.clone(), &mut unordered_delegate).await?; - - struct Nodes { - nodes: LabelIndexedSet, - } - - impl NodeLookup for Nodes { - fn get(&self, label: &T::NodeRef) -> anyhow::Result { - Ok(self - .nodes - .get(label) - .unwrap_or_else(|| { - panic!( - "Should've failed in first traversal if there's a missing node (missing `{}`).", - label - ) - }) - .dupe()) - } - } - - let nodes = Nodes { - nodes: unordered_delegate.nodes, - }; + let graph = Graph::build(nodes, root.clone().into_iter().cloned(), successors).await?; - async_fast_depth_first_postorder_traversal(&nodes, root, delegate).await?; - - Ok(()) + graph.depth_first_postorder_traversal(root.into_iter().cloned(), |node| visit(node.dupe())) } #[cfg(test)] @@ -332,13 +223,12 @@ mod tests { use buck2_core::cells::cell_path::CellPath; use derive_more::Display; use dupe::Dupe; + use dupe::IterDupedExt; use gazebo::prelude::*; - use serde::Serialize; - use serde::Serializer; use super::*; - use crate::query::environment::NodeLabel; use crate::query::environment::QueryTarget; + use crate::query::graph::node::NodeKey; use crate::query::syntax::simple::eval::set::TargetSet; #[derive(Debug, Clone)] @@ -350,15 +240,15 @@ mod tests { #[derive(Debug, Clone, Dupe, Hash, Display, PartialEq, Eq, PartialOrd, Ord)] struct Ref(i64); - impl NodeLabel for Ref {} + impl NodeKey for Ref {} - #[derive(Debug, Display, Serialize)] + #[derive(Debug, Display)] struct Attr(String); impl LabeledNode for Node { - type NodeRef = Ref; + type Key = Ref; - fn node_ref(&self) -> &Self::NodeRef { + fn node_key(&self) -> &Self::Key { &self.0 } } @@ -373,8 +263,8 @@ mod tests { unimplemented!() } - fn deps<'a>(&'a self) -> Box + Send + 'a> { - Box::new(self.1.iter()) + fn deps<'a>(&'a self) -> impl Iterator + Send + 'a { + self.1.iter() } fn special_attrs_for_each) -> Result<(), E>>( @@ -409,68 +299,53 @@ mod tests { unimplemented!() } - fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { + fn exec_deps<'a>(&'a self) -> Box + Send + 'a> { unimplemented!() } - fn target_deps<'a>(&'a self) -> Box + Send + 'a> { + fn target_deps<'a>(&'a self) -> Box + Send + 'a> { unimplemented!() } - - fn call_stack(&self) -> Option { - None - } - - fn attr_to_string_alternate(&self, _attr: &Self::Attr<'_>) -> String { - unimplemented!("not needed for tests") - } - - fn attr_serialize( - &self, - _attr: &Self::Attr<'_>, - _serializer: S, - ) -> Result { - unimplemented!("not needed for tests") - } } struct Graph(HashMap); impl Graph { - fn collecting_delegate<'a>( - &self, - results: &'a mut Vec, - ) -> impl AsyncTraversalDelegate + 'a { - struct Delegate<'a> { - results: &'a mut Vec, - } - - #[async_trait] - impl<'a> AsyncTraversalDelegate for Delegate<'a> { - fn visit(&mut self, target: Node) -> anyhow::Result<()> { - self.results.push(target.0.dupe()); - Ok(()) - } + fn child_visitor<'a>(&self) -> impl AsyncChildVisitor + 'a { + struct ChildVisitorImpl; + impl AsyncChildVisitor for ChildVisitorImpl { async fn for_each_child( - &mut self, + &self, target: &Node, - func: &mut dyn ChildVisitor, + mut func: impl ChildVisitor, ) -> anyhow::Result<()> { for child in &target.1 { - func.visit(child.dupe())?; + func.visit(child)?; } Ok(()) } } - Delegate { results } + ChildVisitorImpl + } + + fn async_node_lookup(&self) -> impl AsyncNodeLookup + '_ { + struct Lookup<'a>(&'a Graph); + + #[async_trait] + impl<'a> AsyncNodeLookup for Lookup<'a> { + async fn get(&self, label: &Ref) -> anyhow::Result { + self.0.get(label) + } + } + + Lookup(self) } } - #[async_trait] - impl AsyncNodeLookup for Graph { - async fn get(&self, label: &Ref) -> anyhow::Result { + impl NodeLookup for Graph { + fn get(&self, label: &Ref) -> anyhow::Result { self.0 .get(label) .ok_or_else(|| anyhow::anyhow!("missing node")) @@ -487,7 +362,7 @@ mod tests { } #[tokio::test] - async fn test_postorder() -> anyhow::Result<()> { + async fn test_async_depth_first_postorder_traversal() -> anyhow::Result<()> { let graph = make_graph(&[ (0, &[1, 2]), (1, &[2, 3, 4]), @@ -496,13 +371,21 @@ mod tests { (4, &[]), ])?; let mut targets = TargetSet::new(); - targets.insert(graph.get(&Ref(0)).await?); + targets.insert(graph.get(&Ref(0))?); let mut results = Vec::new(); { - let mut delegate = graph.collecting_delegate(&mut results); - async_depth_first_postorder_traversal(&graph, targets.iter_names(), &mut delegate) - .await?; + let child_visitor = graph.child_visitor(); + async_depth_first_postorder_traversal( + &graph.async_node_lookup(), + targets.iter_names(), + child_visitor, + |n| { + results.push(n.0); + Ok(()) + }, + ) + .await?; } assert_eq!(results, vec![Ref(4), Ref(3), Ref(2), Ref(1), Ref(0)]); @@ -520,22 +403,72 @@ mod tests { (4, &[]), ])?; let mut targets = TargetSet::new(); - targets.insert(graph.get(&Ref(0)).await?); + targets.insert(graph.get(&Ref(0))?); let mut results0 = Vec::new(); { - let mut delegate = graph.collecting_delegate(&mut results0); - async_depth_limited_traversal(&graph, targets.iter_names(), &mut delegate, 0).await?; + let delegate = graph.child_visitor(); + async_depth_limited_traversal( + &graph.async_node_lookup(), + targets.iter_names(), + delegate, + |n| { + results0.push(n.0); + Ok(()) + }, + 0, + ) + .await?; } assert_eq!(results0, vec![Ref(0)]); let mut results1 = Vec::new(); { - let mut delegate = graph.collecting_delegate(&mut results1); - async_depth_limited_traversal(&graph, targets.iter_names(), &mut delegate, 1).await?; + let delegate = graph.child_visitor(); + async_depth_limited_traversal( + &graph.async_node_lookup(), + targets.iter_names(), + delegate, + |n| { + results1.push(n.0); + Ok(()) + }, + 1, + ) + .await?; } assert_eq!(results1, vec![Ref(0), Ref(1), Ref(2)]); Ok(()) } + + #[tokio::test] + async fn test_async_fast_depth_first_postorder_traversal() -> anyhow::Result<()> { + let graph = make_graph(&[ + (0, &[1, 2]), + (1, &[2, 3, 4]), + (2, &[3, 4]), + (3, &[4]), + (4, &[]), + ])?; + let mut targets = TargetSet::new(); + targets.insert(graph.get(&Ref(0))?); + + let mut results = Vec::new(); + { + async_fast_depth_first_postorder_traversal( + &graph, + targets.iter_names().duped(), + graph.child_visitor(), + |n| { + results.push(n.0); + Ok(()) + }, + ) + .await?; + } + assert_eq!(results, vec![Ref(4), Ref(3), Ref(2), Ref(1), Ref(0)]); + + Ok(()) + } } diff --git a/app/buck2_query_derive/src/codegen.rs b/app/buck2_query_derive/src/codegen.rs index cccc59d2dc385..09e3156641763 100644 --- a/app/buck2_query_derive/src/codegen.rs +++ b/app/buck2_query_derive/src/codegen.rs @@ -206,11 +206,13 @@ fn gen_for_method(parsed: &Parsed, method: &Method) -> syn::Result Some(#func_ty::ref_cast(self) as &dyn QueryBinaryOp<#env_ident>) )); - let method_def = quote_spanned!(method.name.span() => + let struct_def: syn::ItemStruct = syn::parse_quote_spanned! { method.name.span() => #[derive(RefCast)] #[repr(transparent)] - struct #func_ty #impl_generics #where_clause(#self_ty); + struct #func_ty #impl_generics (#self_ty) #where_clause; + }; + let impl_def: syn::ItemImpl = syn::parse_quote_spanned! { method.name.span() => #[async_trait] impl #impl_generics QueryBinaryOp<#env_ident> for #func_ty #ty_generics #where_clause { fn name(&self) -> &'static str { #func_ident_str } @@ -228,6 +230,11 @@ fn gen_for_method(parsed: &Parsed, method: &Method) -> syn::Result + #struct_def + #impl_def ); Ok(MethodCodegen { @@ -244,11 +251,13 @@ fn gen_for_method(parsed: &Parsed, method: &Method) -> syn::Result Some(#func_ty::ref_cast(self) as &dyn QueryFunction<#env_ident>) )); - let method_def = quote_spanned!(method.name.span() => + let struct_def: syn::ItemStruct = syn::parse_quote_spanned! { method.name.span() => #[derive(RefCast)] #[repr(transparent)] - struct #func_ty #impl_generics #where_clause(#self_ty); + struct #func_ty #impl_generics (#self_ty) #where_clause; + }; + let impl_ref: syn::ItemImpl = syn::parse_quote_spanned! { method.name.span() => #[async_trait] impl #impl_generics QueryFunction<#env_ident> for #func_ty #ty_generics #where_clause { fn name(&self) -> &'static str { stringify!(#func_ident) } @@ -275,6 +284,11 @@ fn gen_for_method(parsed: &Parsed, method: &Method) -> syn::Result + #struct_def + #impl_ref ); Ok(MethodCodegen { diff --git a/app/buck2_query_derive/src/parse.rs b/app/buck2_query_derive/src/parse.rs index 9d0e9ff159b72..3bab4b4b9f77b 100644 --- a/app/buck2_query_derive/src/parse.rs +++ b/app/buck2_query_derive/src/parse.rs @@ -288,7 +288,7 @@ impl syn::parse::Parse for QueryModuleArgs { } #[cfg(test)] -mod test { +mod tests { use quote::quote; use syn::parse_quote; use syn::Generics; @@ -350,19 +350,19 @@ mod test { assert_eq!(3, module.methods.len()); - let method = module.methods.get(0).unwrap(); + let method = module.methods.first().unwrap(); assert_eq!("buildfile", method.name.to_string()); let args = &method.args; assert_eq!(1, args.len()); let expected: Type = parse_quote!(TargetSet); - assert_eq!(&expected, &args.get(0).unwrap().ty); + assert_eq!(&expected, &args.first().unwrap().ty); let method = module.methods.get(1).unwrap(); assert_eq!("deps", method.name.to_string()); let args = &method.args; assert_eq!(4, args.len()); let expected: Type = parse_quote!(&QueryEvaluator<'_, Env>); - assert_eq!(&expected, &args.get(0).unwrap().ty); + assert_eq!(&expected, &args.first().unwrap().ty); let expected: Type = parse_quote!(TargetSet); assert_eq!(&expected, &args.get(1).unwrap().ty); let expected: Type = parse_quote!(Option); @@ -375,7 +375,7 @@ mod test { let args = &method.args; assert_eq!(2, args.len()); let expected: Type = parse_quote!(String); - assert_eq!(&expected, &args.get(0).unwrap().ty); + assert_eq!(&expected, &args.first().unwrap().ty); let expected: Type = parse_quote!(TargetSet); assert_eq!(&expected, &args.get(1).unwrap().ty); } diff --git a/app/buck2_query_impls/src/analysis/configured_graph.rs b/app/buck2_query_impls/src/analysis/configured_graph.rs index 4c0c292a70e84..0e93b6e1ec164 100644 --- a/app/buck2_query_impls/src/analysis/configured_graph.rs +++ b/app/buck2_query_impls/src/analysis/configured_graph.rs @@ -32,8 +32,8 @@ use indexmap::IndexMap; use crate::analysis::environment::get_from_template_placeholder_info; use crate::analysis::environment::ConfiguredGraphQueryEnvironmentDelegate; -pub struct AnalysisDiceQueryDelegate<'c> { - pub ctx: &'c DiceComputations, +pub(crate) struct AnalysisDiceQueryDelegate<'c> { + pub(crate) ctx: &'c DiceComputations, } impl<'c> AnalysisDiceQueryDelegate<'c> { @@ -42,9 +42,9 @@ impl<'c> AnalysisDiceQueryDelegate<'c> { } } -pub struct AnalysisConfiguredGraphQueryDelegate<'a> { - pub dice_query_delegate: Arc>, - pub resolved_literals: HashMap, +pub(crate) struct AnalysisConfiguredGraphQueryDelegate<'a> { + pub(crate) dice_query_delegate: Arc>, + pub(crate) resolved_literals: HashMap, } #[async_trait] @@ -93,8 +93,10 @@ impl<'a> ConfiguredGraphQueryEnvironmentDelegate for AnalysisConfiguredGraphQuer ) .await?; - let targets: TargetSet<_> = - targets.into_iter().map(ConfiguredGraphNodeRef).collect(); + let targets: TargetSet<_> = targets + .into_iter() + .map(ConfiguredGraphNodeRef::new) + .collect(); let targets = find_target_nodes(targets, label_to_artifact)?; Ok(Arc::new(targets)) } @@ -146,8 +148,8 @@ fn find_target_nodes( if result.len() == label_to_artifact.len() { return Ok(result); } - for dep in target.0.target_deps() { - let dep = ConfiguredGraphNodeRef(dep.dupe()); + for dep in target.target_deps() { + let dep = ConfiguredGraphNodeRef::new(dep.dupe()); if seen.insert(dep.dupe()) { queue.push_back(dep); } diff --git a/app/buck2_query_impls/src/analysis/environment.rs b/app/buck2_query_impls/src/analysis/environment.rs index 9e8e9d31fa58e..74a204f61ae66 100644 --- a/app/buck2_query_impls/src/analysis/environment.rs +++ b/app/buck2_query_impls/src/analysis/environment.rs @@ -38,11 +38,15 @@ use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersName; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured_node_ref::ConfiguredTargetNodeRefNode; +use buck2_node::nodes::configured_node_ref::ConfiguredTargetNodeRefNodeDeps; use buck2_node::nodes::configured_ref::ConfiguredGraphNodeRef; -use buck2_node::nodes::configured_ref::ConfiguredGraphNodeRefLookup; use buck2_node::query::query_functions::CONFIGURED_GRAPH_QUERY_FUNCTIONS; -use buck2_query::query::environment::LabeledNode; +use buck2_query::query::environment::deps; use buck2_query::query::environment::QueryEnvironment; +use buck2_query::query::environment::TraversalFilter; +use buck2_query::query::graph::dfs::dfs_postorder; +use buck2_query::query::graph::successors::AsyncChildVisitor; use buck2_query::query::syntax::simple::eval::error::QueryError; use buck2_query::query::syntax::simple::eval::file_set::FileSet; use buck2_query::query::syntax::simple::eval::set::TargetSet; @@ -53,11 +57,12 @@ use buck2_query::query::syntax::simple::functions::DefaultQueryFunctionsModule; use buck2_query::query::syntax::simple::functions::QueryFunctions; use buck2_query::query::traversal::async_depth_limited_traversal; use buck2_query::query::traversal::async_fast_depth_first_postorder_traversal; -use buck2_query::query::traversal::AsyncTraversalDelegate; +use buck2_query::query::traversal::NodeLookupId; use buck2_query::query_module; use buck2_query_parser::BinaryOp; use dice::DiceComputations; use dupe::Dupe; +use dupe::IterDupedExt; use indexmap::IndexMap; use starlark::values::UnpackValue; @@ -72,7 +77,7 @@ enum AnalysisQueryError { } #[async_trait] -pub trait ConfiguredGraphQueryEnvironmentDelegate: Send + Sync { +pub(crate) trait ConfiguredGraphQueryEnvironmentDelegate: Send + Sync { fn eval_literal(&self, literal: &str) -> anyhow::Result; async fn get_targets_from_template_placeholder_info( @@ -82,7 +87,7 @@ pub trait ConfiguredGraphQueryEnvironmentDelegate: Send + Sync { ) -> anyhow::Result>; } -pub struct ConfiguredGraphQueryEnvironment<'a> { +pub(crate) struct ConfiguredGraphQueryEnvironment<'a> { delegate: &'a dyn ConfiguredGraphQueryEnvironmentDelegate, } @@ -119,11 +124,11 @@ impl<'a> ConfiguredGraphFunctions<'a> { } impl<'a> ConfiguredGraphQueryEnvironment<'a> { - pub fn new(delegate: &'a dyn ConfiguredGraphQueryEnvironmentDelegate) -> Self { + pub(crate) fn new(delegate: &'a dyn ConfiguredGraphQueryEnvironmentDelegate) -> Self { Self { delegate } } - pub fn functions() -> impl QueryFunctions> { + pub(crate) fn functions() -> impl QueryFunctions> { struct Functions<'a> { defaults: DefaultQueryFunctionsModule>, extra_functions: ConfiguredGraphFunctions<'a>, @@ -202,7 +207,9 @@ impl<'a> QueryEnvironment for ConfiguredGraphQueryEnvironment<'a> { async fn eval_literals(&self, literal: &[&str]) -> anyhow::Result> { let mut result = TargetSet::new(); for lit in literal { - result.insert(ConfiguredGraphNodeRef(self.delegate.eval_literal(lit)?)); + result.insert(ConfiguredGraphNodeRef::new( + self.delegate.eval_literal(lit)?, + )); } Ok(result) } @@ -214,12 +221,14 @@ impl<'a> QueryEnvironment for ConfiguredGraphQueryEnvironment<'a> { async fn dfs_postorder( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { async_fast_depth_first_postorder_traversal( - &ConfiguredGraphNodeRefLookup, - root.iter().map(LabeledNode::node_ref), + &NodeLookupId, + root.iter().duped(), delegate, + visit, ) .await } @@ -227,16 +236,39 @@ impl<'a> QueryEnvironment for ConfiguredGraphQueryEnvironment<'a> { async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()> { - async_depth_limited_traversal(&ConfiguredGraphNodeRefLookup, root.iter(), delegate, depth) - .await + async_depth_limited_traversal(&NodeLookupId, root.iter(), delegate, visit, depth).await } async fn owner(&self, _paths: &FileSet) -> anyhow::Result> { Err(QueryError::FunctionUnimplemented("owner").into()) } + + async fn deps( + &self, + targets: &TargetSet, + depth: Option, + filter: Option<&dyn TraversalFilter>, + ) -> anyhow::Result> { + if depth.is_none() && filter.is_none() { + // TODO(nga): fast lookup with depth too. + let mut deps: TargetSet = TargetSet::new(); + dfs_postorder::( + targets.iter().map(|n| ConfiguredTargetNodeRefNode::new(n)), + ConfiguredTargetNodeRefNodeDeps, + |target| { + deps.insert(ConfiguredGraphNodeRef::new(target.to_node())); + Ok(()) + }, + )?; + Ok(deps) + } else { + deps(self, targets, depth, filter).await + } + } } async fn dice_lookup_transitive_set( @@ -357,7 +389,7 @@ pub(crate) async fn get_from_template_placeholder_info<'x>( Ok(()) }; - match artifact.resolved()? { + match artifact.resolved_artifact(ctx).await? { ResolvedArtifactGroup::Artifact(artifact) => { handle_artifact(&mut label_to_artifact, &artifact)?; } diff --git a/app/buck2_query_impls/src/analysis/eval.rs b/app/buck2_query_impls/src/analysis/eval.rs index 365a4f39175eb..3eb6bc6a8a119 100644 --- a/app/buck2_query_impls/src/analysis/eval.rs +++ b/app/buck2_query_impls/src/analysis/eval.rs @@ -21,7 +21,7 @@ use crate::analysis::configured_graph::AnalysisConfiguredGraphQueryDelegate; use crate::analysis::configured_graph::AnalysisDiceQueryDelegate; use crate::analysis::environment::ConfiguredGraphQueryEnvironment; -pub fn init_eval_analysis_query() { +pub(crate) fn init_eval_analysis_query() { EVAL_ANALYSIS_QUERY.init(|ctx, query, resolved_literals| { Box::pin(eval_analysis_query(ctx, query, resolved_literals)) }); diff --git a/app/buck2_query_impls/src/analysis/evaluator.rs b/app/buck2_query_impls/src/analysis/evaluator.rs index 66a19f5819cff..c5536c88f6462 100644 --- a/app/buck2_query_impls/src/analysis/evaluator.rs +++ b/app/buck2_query_impls/src/analysis/evaluator.rs @@ -9,73 +9,107 @@ //! Implementation of common cquery/uquery pieces. +use anyhow::Context; +use buck2_common::scope::scope_and_collect_with_dispatcher; +use buck2_events::dispatch::EventDispatcher; use buck2_query::query::environment::QueryEnvironment; use buck2_query::query::syntax::simple::eval::evaluator::QueryEvaluator; use buck2_query::query::syntax::simple::eval::literals::extract_target_literals; -use buck2_query::query::syntax::simple::eval::multi_query::process_multi_query; +use buck2_query::query::syntax::simple::eval::multi_query::MultiQueryResult; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; +use buck2_query::query::syntax::simple::eval::values::QueryEvaluationValue; use buck2_query::query::syntax::simple::functions::QueryFunctions; -use buck2_query_parser::placeholder::QUERY_PERCENT_S_PLACEHOLDER; +use buck2_query_parser::multi_query::MaybeMultiQuery; +use buck2_query_parser::multi_query::MultiQueryItem; use futures::Future; -use gazebo::prelude::*; use starlark::collections::SmallSet; -#[derive(Debug, buck2_error::Error)] -enum EvalQueryError { - #[error("Query args supplied without any `%s` placeholder in the query, got args {}", .0.map(|x| format!("`{}`", x)).join(", "))] - ArgsWithoutPlaceholder(Vec), - #[error("Placeholder `%s` in query argument `{0}`")] - PlaceholderInPattern(String), -} - -pub async fn eval_query< +pub(crate) async fn eval_query< F: QueryFunctions, Env: QueryEnvironment, - Fut: Future>, + Fut: Future> + Send, A: AsRef, >( + dispatcher: EventDispatcher, functions: &F, query: &str, query_args: &[A], - environment: impl FnOnce(Vec) -> Fut, + environment: impl Fn(Vec) -> Fut + Send + Sync, ) -> anyhow::Result> { + let query = MaybeMultiQuery::parse(query, query_args)?; + match query { + MaybeMultiQuery::MultiQuery(queries) => { + let results = process_multi_query(dispatcher, functions, environment, &queries).await?; + Ok(QueryEvaluationResult::Multiple(results)) + } + MaybeMultiQuery::SingleQuery(query) => { + let result = eval_single_query(functions, &query, environment).await?; + Ok(QueryEvaluationResult::Single(result)) + } + } +} + +async fn eval_single_query< + F: QueryFunctions, + Env: QueryEnvironment, + Fut: Future>, +>( + functions: &F, + query: &str, + environment: impl Fn(Vec) -> Fut, +) -> anyhow::Result::Target>> +where + F: QueryFunctions, + Env: QueryEnvironment, + Fut: Future>, +{ let mut literals = SmallSet::new(); - if query.contains(QUERY_PERCENT_S_PLACEHOLDER) { - // We'd really like the query args to only be literals (file or target). - // If that didn't work, we'd really like query args to be well-formed expressions. - // Unfortunately Buck1 just substitutes in arbitrarily strings, where the query - // or query_args may not form anything remotely valid. - // We have to be backwards compatible :( - for q in query_args { - let q = q.as_ref(); - if q.contains(QUERY_PERCENT_S_PLACEHOLDER) { - return Err(EvalQueryError::PlaceholderInPattern(q.to_owned()).into()); + extract_target_literals(functions, query, &mut literals)?; + let env = environment(literals.into_iter().collect()).await?; + QueryEvaluator::new(&env, functions).eval_query(query).await +} + +async fn process_multi_query( + dispatcher: EventDispatcher, + functions: &Qf, + env: impl Fn(Vec) -> EnvFut + Send + Sync, + queries: &[MultiQueryItem], +) -> anyhow::Result> +where + Qf: QueryFunctions, + Env: QueryEnvironment, + EnvFut: Future> + Send, +{ + // SAFETY: it is safe as long as we don't forget the future. We don't do that. + let ((), future_results) = unsafe { + scope_and_collect_with_dispatcher(dispatcher, |scope| { + for (i, query) in queries.iter().enumerate() { + let arg: String = query.arg.clone(); + let arg_1: String = query.arg.clone(); + let env = &env; + scope.spawn_cancellable( + async move { + let result = eval_single_query(functions, &query.query, env); + let result = result.await; + (i, arg, result) + }, + move || (i, arg_1, Err(anyhow::anyhow!("future was cancelled"))), + ) } - extract_target_literals( - functions, - &query.replace(QUERY_PERCENT_S_PLACEHOLDER, q), - &mut literals, - )?; - } - let env = environment(literals.into_iter().collect()).await?; - let results = process_multi_query(query, query_args, |input, query| { - let evaluator = QueryEvaluator::new(&env, functions); - async move { (input, evaluator.eval_query(&query).await) } }) - .await; - Ok(QueryEvaluationResult::Multiple(results)) - } else if !query_args.is_empty() { - Err( - EvalQueryError::ArgsWithoutPlaceholder(query_args.map(|s| s.as_ref().to_owned())) - .into(), - ) - } else { - extract_target_literals(functions, query, &mut literals)?; - let env = environment(literals.into_iter().collect()).await?; - Ok(QueryEvaluationResult::Single( - QueryEvaluator::new(&env, functions) - .eval_query(query) - .await?, - )) + .await + }; + + let mut results = Vec::with_capacity(future_results.len()); + for query_result in future_results { + let (i, query, result) = query_result.context("scope_and_collect failed")?; + results.push((i, query, result)); } + results.sort_by_key(|(i, _, _)| *i); + + let map = results + .into_iter() + .map(|(_, query, result)| (query, result)) + .collect(); + Ok(MultiQueryResult(map)) } diff --git a/app/buck2_query_impls/src/analysis/mod.rs b/app/buck2_query_impls/src/analysis/mod.rs index cbbe08fd2354d..8c38f1e0903ac 100644 --- a/app/buck2_query_impls/src/analysis/mod.rs +++ b/app/buck2_query_impls/src/analysis/mod.rs @@ -8,6 +8,6 @@ */ pub(crate) mod configured_graph; -pub mod environment; +pub(crate) mod environment; pub(crate) mod eval; pub(crate) mod evaluator; diff --git a/app/buck2_query_impls/src/aquery/bxl.rs b/app/buck2_query_impls/src/aquery/bxl.rs index 866503be77b3f..925177319a5ee 100644 --- a/app/buck2_query_impls/src/aquery/bxl.rs +++ b/app/buck2_query_impls/src/aquery/bxl.rs @@ -16,6 +16,7 @@ use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; use buck2_build_api::query::bxl::BxlAqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_AQUERY_FUNCTIONS; use buck2_common::dice::cells::HasCellResolver; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::package_boundary::HasPackageBoundaryExceptions; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::configuration::compatibility::MaybeCompatible; @@ -23,7 +24,6 @@ use buck2_core::fs::project::ProjectRoot; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::target::configured_target_label::ConfiguredTargetLabel; -use buck2_core::target::label::TargetLabel; use buck2_query::query::syntax::simple::eval::file_set::FileSet; use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::syntax::simple::eval::values::QueryValue; @@ -50,7 +50,7 @@ fn special_aquery_functions<'v>() -> AqueryFunctions<'v> { } struct BxlAqueryFunctionsImpl { - target_platform: Option, + global_cfg_options: GlobalCfgOptions, project_root: ProjectRoot, working_dir: ProjectRelativePathBuf, } @@ -68,7 +68,7 @@ impl BxlAqueryFunctionsImpl { .await?; let query_data = Arc::new(DiceQueryData::new( - self.target_platform.clone(), + self.global_cfg_options.dupe(), cell_resolver.dupe(), &self.working_dir, self.project_root.dupe(), @@ -259,16 +259,18 @@ impl BxlAqueryFunctions for BxlAqueryFunctionsImpl { } pub(crate) fn init_new_bxl_aquery_functions() { - NEW_BXL_AQUERY_FUNCTIONS.init(|target_platform, project_root, cell_name, cell_resolver| { - Box::pin(async move { - let cell = cell_resolver.get(cell_name)?; - let working_dir = cell.path().as_project_relative_path().to_buf(); + NEW_BXL_AQUERY_FUNCTIONS.init( + |global_cfg_options, project_root, cell_name, cell_resolver| { + Box::pin(async move { + let cell = cell_resolver.get(cell_name)?; + let working_dir = cell.path().as_project_relative_path().to_buf(); - Result::, _>::Ok(Box::new(BxlAqueryFunctionsImpl { - target_platform, - project_root, - working_dir, - })) - }) - }) + Result::, _>::Ok(Box::new(BxlAqueryFunctionsImpl { + global_cfg_options: global_cfg_options.dupe(), + project_root, + working_dir, + })) + }) + }, + ) } diff --git a/app/buck2_query_impls/src/aquery/environment.rs b/app/buck2_query_impls/src/aquery/environment.rs index 47918c5052e98..3cf5eaced1f6f 100644 --- a/app/buck2_query_impls/src/aquery/environment.rs +++ b/app/buck2_query_impls/src/aquery/environment.rs @@ -18,6 +18,7 @@ use buck2_build_api::artifact_groups::ArtifactGroup; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_query::query::environment::QueryEnvironment; +use buck2_query::query::graph::successors::AsyncChildVisitor; use buck2_query::query::syntax::simple::eval::error::QueryError; use buck2_query::query::syntax::simple::eval::file_set::FileSet; use buck2_query::query::syntax::simple::eval::set::TargetSet; @@ -27,7 +28,6 @@ use buck2_query::query::syntax::simple::functions::HasModuleDescription; use buck2_query::query::traversal::async_depth_first_postorder_traversal; use buck2_query::query::traversal::async_depth_limited_traversal; use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::AsyncTraversalDelegate; use dice::DiceComputations; use crate::aquery::functions::AqueryFunctions; @@ -36,7 +36,7 @@ use crate::uquery::environment::QueryLiterals; /// CqueryDelegate resolves information needed by the QueryEnvironment. #[async_trait] -pub trait AqueryDelegate: Send + Sync { +pub(crate) trait AqueryDelegate: Send + Sync { fn cquery_delegate(&self) -> &dyn CqueryDelegate; fn ctx(&self) -> &DiceComputations; @@ -55,13 +55,13 @@ pub trait AqueryDelegate: Send + Sync { ) -> anyhow::Result>; } -pub struct AqueryEnvironment<'c> { +pub(crate) struct AqueryEnvironment<'c> { pub(super) delegate: Arc, literals: Arc + 'c>, } impl<'c> AqueryEnvironment<'c> { - pub fn new( + pub(crate) fn new( delegate: Arc, literals: Arc + 'c>, ) -> Self { @@ -119,7 +119,8 @@ impl<'c> QueryEnvironment for AqueryEnvironment<'c> { async fn dfs_postorder( &self, root: &TargetSet, - traversal_delegate: &mut dyn AsyncTraversalDelegate, + traversal_delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { // TODO(cjhopman): The query nodes deps are going to flatten the tset structure for its deps. In a typical // build graph, a traversal over just the graph of ActionQueryNode ends up being an `O(n)` operation at each @@ -134,6 +135,7 @@ impl<'c> QueryEnvironment for AqueryEnvironment<'c> { }, root.iter_names(), traversal_delegate, + visit, ) .await } @@ -141,7 +143,8 @@ impl<'c> QueryEnvironment for AqueryEnvironment<'c> { async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()> { // TODO(cjhopman): See above. @@ -152,6 +155,7 @@ impl<'c> QueryEnvironment for AqueryEnvironment<'c> { }, root.iter_names(), delegate, + visit, depth, ) .await diff --git a/app/buck2_query_impls/src/aquery/evaluator.rs b/app/buck2_query_impls/src/aquery/evaluator.rs index b6ffd30769e6b..3a8b9cd2e1bd5 100644 --- a/app/buck2_query_impls/src/aquery/evaluator.rs +++ b/app/buck2_query_impls/src/aquery/evaluator.rs @@ -11,8 +11,9 @@ use std::sync::Arc; use buck2_build_api::actions::query::ActionQueryNode; +use buck2_common::events::HasEvents; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; use dice::DiceComputations; use dupe::Dupe; @@ -37,18 +38,28 @@ impl AqueryEvaluator<'_> { ) -> anyhow::Result> { let functions = aquery_functions(); - eval_query(&functions, query, query_args, async move |literals| { - let resolved_literals = PreresolvedQueryLiterals::pre_resolve( - &**self.dice_query_delegate.query_data(), - &literals, - self.dice_query_delegate.ctx(), - ) - .await; - Ok(AqueryEnvironment::new( - self.dice_query_delegate.dupe(), - Arc::new(resolved_literals), - )) - }) + eval_query( + self.dice_query_delegate + .ctx() + .per_transaction_data() + .get_dispatcher() + .dupe(), + &functions, + query, + query_args, + async move |literals| { + let resolved_literals = PreresolvedQueryLiterals::pre_resolve( + &**self.dice_query_delegate.query_data(), + &literals, + self.dice_query_delegate.ctx(), + ) + .await; + Ok(AqueryEnvironment::new( + self.dice_query_delegate.dupe(), + Arc::new(resolved_literals), + )) + }, + ) .await } } @@ -58,10 +69,10 @@ impl AqueryEvaluator<'_> { pub(crate) async fn get_aquery_evaluator<'a, 'c: 'a>( ctx: &'c DiceComputations, working_dir: &'a ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result> { let dice_query_delegate = - get_dice_aquery_delegate(ctx, working_dir, global_target_platform).await?; + get_dice_aquery_delegate(ctx, working_dir, global_cfg_options).await?; Ok(AqueryEvaluator { dice_query_delegate, }) @@ -71,10 +82,9 @@ pub(crate) async fn get_aquery_evaluator<'a, 'c: 'a>( pub(crate) async fn get_dice_aquery_delegate<'a, 'c: 'a>( ctx: &'c DiceComputations, working_dir: &'a ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result>> { - let dice_query_delegate = - get_dice_query_delegate(ctx, working_dir, global_target_platform).await?; + let dice_query_delegate = get_dice_query_delegate(ctx, working_dir, global_cfg_options).await?; let dice_query_delegate = Arc::new(DiceAqueryDelegate::new(dice_query_delegate).await?); Ok(dice_query_delegate) } diff --git a/app/buck2_query_impls/src/aquery/find_matching_action.rs b/app/buck2_query_impls/src/aquery/find_matching_action.rs index 4775d7727a620..aa82a31599774 100644 --- a/app/buck2_query_impls/src/aquery/find_matching_action.rs +++ b/app/buck2_query_impls/src/aquery/find_matching_action.rs @@ -13,18 +13,27 @@ use buck2_artifact::artifact::provide_outputs::ProvideOutputs; use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::actions::query::FIND_MATCHING_ACTION; use buck2_build_api::analysis::AnalysisResult; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use dice::DiceComputations; +use dupe::Dupe; use tracing::debug; use crate::aquery::evaluator::get_dice_aquery_delegate; +// Given the buckout path, how do we search actions? +enum ActionKeyMatch<'v> { + // This action key exactly produces the output path. + Exact(&'v ActionKey), + // Builds an output that is in the path. + OutputsOf(&'v BuildArtifact), +} + fn check_output_path<'v>( build_artifact: &'v BuildArtifact, path_to_check: &'v ForwardRelativePathBuf, -) -> anyhow::Result> { +) -> anyhow::Result>> { let path = build_artifact.get_path().path(); debug!( @@ -32,8 +41,12 @@ fn check_output_path<'v>( path, path_to_check ); - if path_to_check.starts_with(path_to_check) { - Ok(Some(build_artifact.key())) + let key = build_artifact.key(); + + if path_to_check == path { + Ok(Some(ActionKeyMatch::Exact(key))) + } else if path_to_check.starts_with(path) { + Ok(Some(ActionKeyMatch::OutputsOf(build_artifact))) } else { Ok(None) } @@ -42,27 +55,55 @@ fn check_output_path<'v>( async fn find_matching_action( ctx: &DiceComputations, working_dir: &ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, analysis: &AnalysisResult, path_after_target_name: ForwardRelativePathBuf, ) -> anyhow::Result> { let dice_aquery_delegate = - get_dice_aquery_delegate(ctx, working_dir, global_target_platform.clone()).await?; + get_dice_aquery_delegate(ctx, working_dir, global_cfg_options.dupe()).await?; for entry in analysis.iter_deferreds() { match provider::request_value::(entry.as_complex()) { Some(outputs) => { let outputs = outputs.0?; + // Try to find exact path match first. If there are no exact matches, try to find an action + // that starts with the relevant part of the output path (this case is for targets that declare + // directories as outputs). + // + // FIXME(@wendyy): If we've iterated over all build artifacts and still haven't found an exact + // action key match, then return a possible action key with the shortest path. This can happen + // if a target declared an output directory instead of an artifact. As a best effort, we keep + // track of the possible build artifact with the shortest path to try find the action that produced + // the top-most directory. To fix this properly, we would need to let the action key or build + // artifact itself know if the output was a directory, which is nontrivial. + let mut maybe_match: Option<&BuildArtifact> = None; for build_artifact in &outputs { match check_output_path(build_artifact, &path_after_target_name)? { - Some(action_key) => { - return Ok(Some( - dice_aquery_delegate.get_action_node(action_key).await?, - )); - } + Some(action_key_match) => match action_key_match { + ActionKeyMatch::Exact(key) => { + return Ok(Some(dice_aquery_delegate.get_action_node(key).await?)); + } + ActionKeyMatch::OutputsOf(artifact) => match maybe_match { + Some(maybe) => { + if artifact.get_path().len() < maybe.get_path().len() { + maybe_match = Some(artifact); + } + } + None => maybe_match = Some(artifact), + }, + }, None => (), } } + + match maybe_match { + Some(maybe) => { + return Ok(Some( + dice_aquery_delegate.get_action_node(maybe.key()).await?, + )); + } + None => (), + } } None => debug!("Could not extract outputs from deferred table entry"), } @@ -72,11 +113,11 @@ async fn find_matching_action( pub(crate) fn init_find_matching_action() { FIND_MATCHING_ACTION.init( - |ctx, working_dir, global_target_platform, analysis, path_after_target_name| { + |ctx, working_dir, global_cfg_options, analysis, path_after_target_name| { Box::pin(find_matching_action( ctx, working_dir, - global_target_platform, + global_cfg_options, analysis, path_after_target_name, )) diff --git a/app/buck2_query_impls/src/aquery/functions.rs b/app/buck2_query_impls/src/aquery/functions.rs index 96aef5794e1a0..a99314377a33d 100644 --- a/app/buck2_query_impls/src/aquery/functions.rs +++ b/app/buck2_query_impls/src/aquery/functions.rs @@ -26,7 +26,7 @@ use buck2_query_parser::BinaryOp; use crate::aquery::environment::AqueryEnvironment; -pub fn aquery_functions<'a>() -> impl QueryFunctions> { +pub(crate) fn aquery_functions<'a>() -> impl QueryFunctions> { struct Functions<'a> { defaults: DefaultQueryFunctionsModule>, extra_functions: AqueryFunctions<'a>, diff --git a/app/buck2_query_impls/src/aquery/mod.rs b/app/buck2_query_impls/src/aquery/mod.rs index 7ed1453b7bee2..5dbef6c8d3604 100644 --- a/app/buck2_query_impls/src/aquery/mod.rs +++ b/app/buck2_query_impls/src/aquery/mod.rs @@ -7,8 +7,8 @@ * of this source tree. */ -pub mod bxl; -pub mod environment; -pub mod evaluator; +pub(crate) mod bxl; +pub(crate) mod environment; +pub(crate) mod evaluator; pub(crate) mod find_matching_action; -pub mod functions; +pub(crate) mod functions; diff --git a/app/buck2_query_impls/src/cquery/bxl.rs b/app/buck2_query_impls/src/cquery/bxl.rs index f8987a94c56b7..8a4726ebc840d 100644 --- a/app/buck2_query_impls/src/cquery/bxl.rs +++ b/app/buck2_query_impls/src/cquery/bxl.rs @@ -14,12 +14,12 @@ use buck2_build_api::query::bxl::BxlCqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_CQUERY_FUNCTIONS; use buck2_build_api::query::oneshot::CqueryOwnerBehavior; use buck2_common::dice::cells::HasCellResolver; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::package_boundary::HasPackageBoundaryExceptions; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::fs::project::ProjectRoot; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; -use buck2_core::target::label::TargetLabel; use buck2_node::configured_universe::CqueryUniverse; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_query::query::syntax::simple::eval::file_set::FileSet; @@ -39,7 +39,7 @@ fn cquery_functions<'v>() -> DefaultQueryFunctions> { } struct BxlCqueryFunctionsImpl { - target_platform: Option, + global_cfg_options: GlobalCfgOptions, project_root: ProjectRoot, working_dir: ProjectRelativePathBuf, } @@ -57,7 +57,7 @@ impl BxlCqueryFunctionsImpl { .await?; let query_data = Arc::new(DiceQueryData::new( - self.target_platform.dupe(), + self.global_cfg_options.dupe(), cell_resolver.dupe(), &self.working_dir, self.project_root.dupe(), @@ -78,7 +78,7 @@ impl BxlCqueryFunctionsImpl { universe: Option<&TargetSet>, ) -> anyhow::Result> { let universe = match universe { - Some(u) => Some(CqueryUniverse::build(u).await?), + Some(u) => Some(CqueryUniverse::build(u)?), None => None, }; let literals = dice_query_delegate.query_data().dupe(); @@ -209,18 +209,20 @@ impl BxlCqueryFunctions for BxlCqueryFunctionsImpl { } pub(crate) fn init_new_bxl_cquery_functions() { - NEW_BXL_CQUERY_FUNCTIONS.init(|target_platform, project_root, cell_name, cell_resolver| { - Box::pin(async move { - let cell = cell_resolver.get(cell_name)?; - // TODO(nga): working as as cell root is not right. - // Should be either the project root or user's current working directory. - let working_dir = cell.path().as_project_relative_path().to_buf(); - - Result::, _>::Ok(Box::new(BxlCqueryFunctionsImpl { - target_platform, - project_root, - working_dir, - })) - }) - }) + NEW_BXL_CQUERY_FUNCTIONS.init( + |global_cfg_options: GlobalCfgOptions, project_root, cell_name, cell_resolver| { + Box::pin(async move { + let cell = cell_resolver.get(cell_name)?; + // TODO(nga): working as as cell root is not right. + // Should be either the project root or user's current working directory. + let working_dir = cell.path().as_project_relative_path().to_buf(); + + Result::, _>::Ok(Box::new(BxlCqueryFunctionsImpl { + global_cfg_options, + project_root, + working_dir, + })) + }) + }, + ) } diff --git a/app/buck2_query_impls/src/cquery/environment.rs b/app/buck2_query_impls/src/cquery/environment.rs index 5505be80f6eaf..4d33f14a17567 100644 --- a/app/buck2_query_impls/src/cquery/environment.rs +++ b/app/buck2_query_impls/src/cquery/environment.rs @@ -19,7 +19,14 @@ use buck2_core::target::label::TargetLabel; use buck2_events::dispatch::console_message; use buck2_node::configured_universe::CqueryUniverse; use buck2_node::nodes::configured::ConfiguredTargetNode; +use buck2_node::nodes::configured_node_ref::ConfiguredTargetNodeRefNode; +use buck2_node::nodes::configured_node_ref::ConfiguredTargetNodeRefNodeDeps; +use buck2_query::query::environment::deps; use buck2_query::query::environment::QueryEnvironment; +use buck2_query::query::environment::QueryEnvironmentAsNodeLookup; +use buck2_query::query::environment::TraversalFilter; +use buck2_query::query::graph::dfs::dfs_postorder; +use buck2_query::query::graph::successors::AsyncChildVisitor; use buck2_query::query::syntax::simple::eval::file_set::FileSet; use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::syntax::simple::functions::docs::QueryEnvironmentDescription; @@ -27,8 +34,6 @@ use buck2_query::query::syntax::simple::functions::DefaultQueryFunctionsModule; use buck2_query::query::syntax::simple::functions::HasModuleDescription; use buck2_query::query::traversal::async_depth_first_postorder_traversal; use buck2_query::query::traversal::async_depth_limited_traversal; -use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::AsyncTraversalDelegate; use dice::DiceComputations; use dupe::Dupe; use tracing::warn; @@ -46,7 +51,7 @@ enum CqueryError { /// CqueryDelegate resolves information needed by the QueryEnvironment. #[async_trait] -pub trait CqueryDelegate: Send + Sync { +pub(crate) trait CqueryDelegate: Send + Sync { fn uquery_delegate(&self) -> &dyn UqueryDelegate; async fn get_node_for_target( @@ -72,7 +77,7 @@ pub trait CqueryDelegate: Send + Sync { fn ctx(&self) -> &DiceComputations; } -pub struct CqueryEnvironment<'c> { +pub(crate) struct CqueryEnvironment<'c> { delegate: &'c dyn CqueryDelegate, literals: Arc + 'c>, // TODO(nga): BXL `cquery` function does not provides us the universe. @@ -86,7 +91,7 @@ pub struct CqueryEnvironment<'c> { } impl<'c> CqueryEnvironment<'c> { - pub fn new( + pub(crate) fn new( delegate: &'c dyn CqueryDelegate, literals: Arc + 'c>, universe: Option, @@ -222,18 +227,33 @@ impl<'c> QueryEnvironment for CqueryEnvironment<'c> { async fn dfs_postorder( &self, root: &TargetSet, - traversal_delegate: &mut dyn AsyncTraversalDelegate, + traversal_delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { - async_depth_first_postorder_traversal(self, root.iter_names(), traversal_delegate).await + async_depth_first_postorder_traversal( + &QueryEnvironmentAsNodeLookup { env: self }, + root.iter_names(), + traversal_delegate, + visit, + ) + .await } async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()> { - async_depth_limited_traversal(self, root.iter_names(), delegate, depth).await + async_depth_limited_traversal( + &QueryEnvironmentAsNodeLookup { env: self }, + root.iter_names(), + delegate, + visit, + depth, + ) + .await } async fn allbuildfiles(&self, universe: &TargetSet) -> anyhow::Result { @@ -259,11 +279,28 @@ impl<'c> QueryEnvironment for CqueryEnvironment<'c> { } Ok(result) } -} -#[async_trait] -impl<'a> AsyncNodeLookup for CqueryEnvironment<'a> { - async fn get(&self, label: &ConfiguredTargetLabel) -> anyhow::Result { - self.get_node(label).await + async fn deps( + &self, + targets: &TargetSet, + depth: Option, + filter: Option<&dyn TraversalFilter>, + ) -> anyhow::Result> { + if depth.is_none() && filter.is_none() { + // TODO(nga): fast lookup with depth too. + + let mut deps = TargetSet::new(); + dfs_postorder::( + targets.iter().map(ConfiguredTargetNodeRefNode::new), + ConfiguredTargetNodeRefNodeDeps, + |target| { + deps.insert_unique_unchecked(target.to_node()); + Ok(()) + }, + )?; + Ok(deps) + } else { + deps(self, targets, depth, filter).await + } } } diff --git a/app/buck2_query_impls/src/cquery/evaluator.rs b/app/buck2_query_impls/src/cquery/evaluator.rs index e024b7b6ece39..cfab1cff09389 100644 --- a/app/buck2_query_impls/src/cquery/evaluator.rs +++ b/app/buck2_query_impls/src/cquery/evaluator.rs @@ -12,8 +12,9 @@ use std::sync::Arc; use buck2_build_api::query::oneshot::CqueryOwnerBehavior; +use buck2_common::events::HasEvents; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_events::dispatch::console_message; use buck2_node::configured_universe::CqueryUniverse; use buck2_node::nodes::configured::ConfiguredTargetNode; @@ -34,20 +35,20 @@ use crate::uquery::environment::PreresolvedQueryLiterals; use crate::uquery::environment::QueryLiterals; use crate::uquery::environment::UqueryDelegate; -pub struct CqueryEvaluator<'c> { +pub(crate) struct CqueryEvaluator<'c> { dice_query_delegate: DiceQueryDelegate<'c>, functions: DefaultQueryFunctionsModule>, owner_behavior: CqueryOwnerBehavior, } impl CqueryEvaluator<'_> { - pub async fn eval_query, U: AsRef>( + pub(crate) async fn eval_query, U: AsRef + Send + Sync>( &self, query: &str, query_args: &[A], target_universe: Option<&[U]>, ) -> anyhow::Result> { - eval_query(&self.functions, query, query_args, async move |literals| { + eval_query(self.dice_query_delegate.ctx().per_transaction_data().get_dispatcher().dupe(), &self.functions, query, query_args, async move |literals| { let (universe, resolved_literals) = match target_universe { None => { if literals.is_empty() { @@ -89,20 +90,19 @@ pub(crate) async fn preresolve_literals_and_build_universe( let resolved_literals = PreresolvedQueryLiterals::pre_resolve(dice_query_data, literals, dice_query_delegate.ctx()) .await; - let universe = CqueryUniverse::build(&resolved_literals.literals()?).await?; + let universe = CqueryUniverse::build(&resolved_literals.literals()?)?; Ok((universe, resolved_literals)) } /// Evaluates some query expression. TargetNodes are resolved via the interpreter from /// the provided DiceCtx. -pub async fn get_cquery_evaluator<'a, 'c: 'a>( +pub(crate) async fn get_cquery_evaluator<'a, 'c: 'a>( ctx: &'c DiceComputations, working_dir: &'a ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, owner_behavior: CqueryOwnerBehavior, ) -> anyhow::Result> { - let dice_query_delegate = - get_dice_query_delegate(ctx, working_dir, global_target_platform).await?; + let dice_query_delegate = get_dice_query_delegate(ctx, working_dir, global_cfg_options).await?; let functions = DefaultQueryFunctionsModule::new(); Ok(CqueryEvaluator { dice_query_delegate, @@ -129,7 +129,7 @@ async fn resolve_literals_in_universe, U: AsRef>( .eval_literals(&refs, dice_query_delegate.ctx()) .await?; - let universe = CqueryUniverse::build(&universe_resolved).await?; + let universe = CqueryUniverse::build(&universe_resolved)?; // capture a reference so the ref can be moved into the future below. let universe_ref = &universe; diff --git a/app/buck2_query_impls/src/cquery/mod.rs b/app/buck2_query_impls/src/cquery/mod.rs index 9c43548eb0f28..6849c9eec9575 100644 --- a/app/buck2_query_impls/src/cquery/mod.rs +++ b/app/buck2_query_impls/src/cquery/mod.rs @@ -8,5 +8,5 @@ */ pub(crate) mod bxl; -pub mod environment; -pub mod evaluator; +pub(crate) mod environment; +pub(crate) mod evaluator; diff --git a/app/buck2_query_impls/src/dice/aquery.rs b/app/buck2_query_impls/src/dice/aquery.rs index 133b7533c7f77..136741c0212b4 100644 --- a/app/buck2_query_impls/src/dice/aquery.rs +++ b/app/buck2_query_impls/src/dice/aquery.rs @@ -27,6 +27,7 @@ use buck2_build_api::artifact_groups::ArtifactGroup; use buck2_build_api::artifact_groups::ResolvedArtifactGroup; use buck2_build_api::artifact_groups::TransitiveSetProjectionKey; use buck2_build_api::deferred::calculation::DeferredCalculation; +use buck2_build_api::keep_going; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::fs::artifact_path_resolver::ArtifactFs; use buck2_core::pattern::ParsedPattern; @@ -161,12 +162,20 @@ async fn convert_inputs<'c, 'a, Iter: IntoIterator>( node_cache: DiceAqueryNodesCache, inputs: Iter, ) -> anyhow::Result> { + let resolved_artifact_futs: FuturesOrdered<_> = inputs + .into_iter() + .map(|input| async { input.resolved_artifact(ctx).await }) + .collect(); + + let resolved_artifacts: Vec<_> = + tokio::task::unconstrained(keep_going::try_join_all(ctx, resolved_artifact_futs)).await?; + let (artifacts, projections): (Vec<_>, Vec<_>) = Itertools::partition_map( - inputs + resolved_artifacts .into_iter() - .filter_map(|input| match input.assert_resolved() { + .filter_map(|resolved_artifact| match resolved_artifact { ResolvedArtifactGroup::Artifact(a) => { - a.action_key().map(|k| Either::Left(k.clone())) + a.action_key().map(|a| Either::Left(a.clone())) } ResolvedArtifactGroup::TransitiveSetProjection(key) => Some(Either::Right(key)), }), @@ -275,11 +284,11 @@ impl<'c> DiceAqueryDelegate<'c> { }) } - pub fn query_data(&self) -> &Arc { + pub(crate) fn query_data(&self) -> &Arc { &self.query_data } - pub async fn get_action_node(&self, key: &ActionKey) -> anyhow::Result { + pub(crate) async fn get_action_node(&self, key: &ActionKey) -> anyhow::Result { get_action_node( self.query_data.nodes_cache.dupe(), self.base_delegate.ctx(), @@ -396,7 +405,7 @@ impl QueryLiterals for AqueryData { let configured_label = dice .get_configured_provider_label( &label, - self.delegate_query_data.global_target_platform(), + self.delegate_query_data.global_cfg_options(), ) .await?; diff --git a/app/buck2_query_impls/src/dice/mod.rs b/app/buck2_query_impls/src/dice/mod.rs index 9c5627231e118..6241cee930d2c 100644 --- a/app/buck2_query_impls/src/dice/mod.rs +++ b/app/buck2_query_impls/src/dice/mod.rs @@ -15,6 +15,7 @@ use buck2_build_api::configure_targets::load_compatible_patterns; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::data::HasIoProvider; use buck2_common::dice::file_ops::HasFileOps; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::package_boundary::HasPackageBoundaryExceptions; use buck2_common::package_boundary::PackageBoundaryExceptions; use buck2_common::package_listing::dice::HasPackageListingResolver; @@ -63,7 +64,7 @@ use crate::cquery::environment::CqueryDelegate; use crate::uquery::environment::QueryLiterals; use crate::uquery::environment::UqueryDelegate; -pub mod aquery; +pub(crate) mod aquery; #[derive(Debug, buck2_error::Error)] enum LiteralParserError { @@ -135,21 +136,21 @@ impl LiteralParser { /// A Uquery delegate that resolves TargetNodes with the provided /// InterpreterCalculation. -pub struct DiceQueryDelegate<'c> { +pub(crate) struct DiceQueryDelegate<'c> { ctx: &'c DiceComputations, cell_resolver: CellResolver, query_data: Arc, package_boundary_exceptions: Arc, } -pub struct DiceQueryData { +pub(crate) struct DiceQueryData { literal_parser: LiteralParser, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, } impl DiceQueryData { - pub fn new( - global_target_platform: Option, + pub(crate) fn new( + global_cfg_options: GlobalCfgOptions, cell_resolver: CellResolver, working_dir: &ProjectRelativePath, project_root: ProjectRoot, @@ -172,7 +173,7 @@ impl DiceQueryData { cell_alias_resolver, target_alias_resolver, }, - global_target_platform, + global_cfg_options, }) } @@ -180,13 +181,13 @@ impl DiceQueryData { &self.literal_parser } - pub(crate) fn global_target_platform(&self) -> Option<&TargetLabel> { - self.global_target_platform.as_ref() + pub(crate) fn global_cfg_options(&self) -> &GlobalCfgOptions { + &self.global_cfg_options } } impl<'c> DiceQueryDelegate<'c> { - pub fn new( + pub(crate) fn new( ctx: &'c DiceComputations, cell_resolver: CellResolver, package_boundary_exceptions: Arc, @@ -290,7 +291,7 @@ impl<'c> CqueryDelegate for DiceQueryDelegate<'c> { ) -> anyhow::Result> { let target = self .ctx - .get_configured_target(target, self.query_data.global_target_platform.as_ref()) + .get_configured_target(target, self.query_data.global_cfg_options()) .await?; Ok(self.ctx.get_configured_target_node(&target).await?) } @@ -319,7 +320,7 @@ impl<'c> CqueryDelegate for DiceQueryDelegate<'c> { target: &TargetLabel, ) -> anyhow::Result { self.ctx - .get_configured_target(target, self.query_data.global_target_platform.as_ref()) + .get_configured_target(target, self.query_data.global_cfg_options()) .await } @@ -339,7 +340,7 @@ impl QueryLiterals for DiceQueryData { load_compatible_patterns( ctx, parsed_patterns, - self.global_target_platform.dupe(), + &self.global_cfg_options, MissingTargetBehavior::Fail, ) .await @@ -367,7 +368,7 @@ impl QueryLiterals for DiceQueryData { pub(crate) async fn get_dice_query_delegate<'a, 'c: 'a>( ctx: &'c DiceComputations, working_dir: &'a ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result> { let cell_resolver = ctx.get_cell_resolver().await?; let package_boundary_exceptions = ctx.get_package_boundary_exceptions().await?; @@ -384,7 +385,7 @@ pub(crate) async fn get_dice_query_delegate<'a, 'c: 'a>( cell_resolver.dupe(), package_boundary_exceptions, Arc::new(DiceQueryData::new( - global_target_platform, + global_cfg_options, cell_resolver, working_dir, project_root, diff --git a/app/buck2_query_impls/src/frontend.rs b/app/buck2_query_impls/src/frontend.rs index e53eb37de00cf..8fd14212627d3 100644 --- a/app/buck2_query_impls/src/frontend.rs +++ b/app/buck2_query_impls/src/frontend.rs @@ -12,8 +12,8 @@ use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::query::oneshot::CqueryOwnerBehavior; use buck2_build_api::query::oneshot::QueryFrontend; use buck2_build_api::query::oneshot::QUERY_FRONTEND; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_node::configured_universe::CqueryUniverse; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_node::nodes::unconfigured::TargetNode; @@ -40,9 +40,9 @@ impl QueryFrontend for QueryFrontendImpl { working_dir: &ProjectRelativePath, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result> { - let evaluator = get_uquery_evaluator(ctx, working_dir, global_target_platform).await?; + let evaluator = get_uquery_evaluator(ctx, working_dir, global_cfg_options).await?; evaluator.eval_query(query, query_args).await } @@ -54,11 +54,11 @@ impl QueryFrontend for QueryFrontendImpl { owner_behavior: CqueryOwnerBehavior, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, target_universe: Option<&[String]>, ) -> anyhow::Result> { let evaluator = - get_cquery_evaluator(ctx, working_dir, global_target_platform, owner_behavior).await?; + get_cquery_evaluator(ctx, working_dir, global_cfg_options, owner_behavior).await?; // TODO(nga): this should support configured target patterns // similarly to what we do for `build` command. @@ -77,9 +77,9 @@ impl QueryFrontend for QueryFrontendImpl { working_dir: &ProjectRelativePath, query: &str, query_args: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result> { - let evaluator = get_aquery_evaluator(ctx, working_dir, global_target_platform).await?; + let evaluator = get_aquery_evaluator(ctx, working_dir, global_cfg_options).await?; evaluator.eval_query(query, query_args).await } @@ -89,9 +89,9 @@ impl QueryFrontend for QueryFrontendImpl { ctx: &DiceComputations, cwd: &ProjectRelativePath, literals: &[String], - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result { - let query_delegate = get_dice_query_delegate(ctx, cwd, global_target_platform).await?; + let query_delegate = get_dice_query_delegate(ctx, cwd, global_cfg_options).await?; Ok(preresolve_literals_and_build_universe( &query_delegate, query_delegate.query_data(), diff --git a/app/buck2_query_impls/src/lib.rs b/app/buck2_query_impls/src/lib.rs index ac71e5c3d26e5..8a1ab65a2985c 100644 --- a/app/buck2_query_impls/src/lib.rs +++ b/app/buck2_query_impls/src/lib.rs @@ -13,13 +13,13 @@ use std::sync::Once; -pub mod analysis; -pub mod aquery; -pub mod cquery; +pub(crate) mod analysis; +pub(crate) mod aquery; +pub(crate) mod cquery; mod description; -pub mod dice; -pub mod frontend; -pub mod uquery; +pub(crate) mod dice; +pub(crate) mod frontend; +pub(crate) mod uquery; pub fn init_late_bindings() { static ONCE: Once = Once::new(); diff --git a/app/buck2_query_impls/src/uquery/bxl.rs b/app/buck2_query_impls/src/uquery/bxl.rs index 5ee8ecef249e5..86947680ee0b3 100644 --- a/app/buck2_query_impls/src/uquery/bxl.rs +++ b/app/buck2_query_impls/src/uquery/bxl.rs @@ -13,6 +13,7 @@ use async_trait::async_trait; use buck2_build_api::query::bxl::BxlUqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_UQUERY_FUNCTIONS; use buck2_common::dice::cells::HasCellResolver; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::package_boundary::HasPackageBoundaryExceptions; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::fs::project::ProjectRoot; @@ -52,7 +53,7 @@ impl BxlUqueryFunctionsImpl { .await?; let query_data = Arc::new(DiceQueryData::new( - None, + GlobalCfgOptions::default(), cell_resolver.dupe(), &self.working_dir, self.project_root.dupe(), diff --git a/app/buck2_query_impls/src/uquery/environment.rs b/app/buck2_query_impls/src/uquery/environment.rs index 4f54026a7e7e6..9319c0e0c6569 100644 --- a/app/buck2_query_impls/src/uquery/environment.rs +++ b/app/buck2_query_impls/src/uquery/environment.rs @@ -24,10 +24,12 @@ use buck2_core::pattern::pattern_type::TargetPatternExtra; use buck2_core::target::label::TargetLabel; use buck2_node::nodes::eval_result::EvaluationResult; use buck2_node::nodes::unconfigured::TargetNode; -use buck2_query::query::environment::LabeledNode; -use buck2_query::query::environment::NodeLabel; use buck2_query::query::environment::QueryEnvironment; +use buck2_query::query::environment::QueryEnvironmentAsNodeLookup; use buck2_query::query::environment::QueryTarget; +use buck2_query::query::graph::node::LabeledNode; +use buck2_query::query::graph::node::NodeKey; +use buck2_query::query::graph::successors::AsyncChildVisitor; use buck2_query::query::syntax::simple::eval::error::QueryError; use buck2_query::query::syntax::simple::eval::file_set::FileNode; use buck2_query::query::syntax::simple::eval::file_set::FileSet; @@ -38,7 +40,6 @@ use buck2_query::query::syntax::simple::functions::HasModuleDescription; use buck2_query::query::traversal::async_depth_first_postorder_traversal; use buck2_query::query::traversal::async_depth_limited_traversal; use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::AsyncTraversalDelegate; use buck2_query::query::traversal::ChildVisitor; use derive_more::Display; use dice::DiceComputations; @@ -67,13 +68,9 @@ enum RBuildFilesError { CellMissingBuildFileNames(CellName), } -pub enum SpecialAttr { - String(String), -} - /// UqueryDelegate resolves information needed by the QueryEnvironment. #[async_trait] -pub trait UqueryDelegate: Send + Sync { +pub(crate) trait UqueryDelegate: Send + Sync { /// Returns the EvaluationResult for evaluation of the buildfile. async fn eval_build_file( &self, @@ -102,7 +99,7 @@ pub trait UqueryDelegate: Send + Sync { } #[async_trait] -pub trait QueryLiterals: Send + Sync { +pub(crate) trait QueryLiterals: Send + Sync { async fn eval_literals( &self, literals: &[&str], @@ -110,21 +107,23 @@ pub trait QueryLiterals: Send + Sync { ) -> anyhow::Result>; } -pub struct UqueryEnvironment<'c> { +pub(crate) struct UqueryEnvironment<'c> { delegate: &'c dyn UqueryDelegate, literals: Arc + 'c>, } -pub struct PreresolvedQueryLiterals { +pub(crate) struct PreresolvedQueryLiterals { resolved_literals: HashMap>>, } impl PreresolvedQueryLiterals { - pub fn new(resolved_literals: HashMap>>) -> Self { + pub(crate) fn new( + resolved_literals: HashMap>>, + ) -> Self { Self { resolved_literals } } - pub async fn pre_resolve( + pub(crate) async fn pre_resolve( base: &dyn QueryLiterals, literals: &[String], dice: &DiceComputations, @@ -140,7 +139,7 @@ impl PreresolvedQueryLiterals { } /// All the literals, or error if resolution of any failed. - pub fn literals(&self) -> anyhow::Result> { + pub(crate) fn literals(&self) -> anyhow::Result> { let mut literals = TargetSet::new(); for result in self.resolved_literals.values() { literals.extend(result.as_ref().map_err(|e| e.dupe())?); @@ -173,7 +172,7 @@ impl QueryLiterals for PreresolvedQueryLiterals { } impl<'c> UqueryEnvironment<'c> { - pub fn new( + pub(crate) fn new( delegate: &'c dyn UqueryDelegate, literals: Arc + 'c>, ) -> Self { @@ -194,7 +193,7 @@ impl<'c> UqueryEnvironment<'c> { .await .with_context(|| format!("Error looking up `{}``", target))?; let node = package.resolve_target(target.name())?; - Ok(node.dupe()) + Ok(node.to_owned()) } } @@ -229,18 +228,33 @@ impl<'c> QueryEnvironment for UqueryEnvironment<'c> { async fn dfs_postorder( &self, root: &TargetSet, - traversal_delegate: &mut dyn AsyncTraversalDelegate, + traversal_delegate: impl AsyncChildVisitor, + visit: impl FnMut(TargetNode) -> anyhow::Result<()> + Send, ) -> anyhow::Result<()> { - async_depth_first_postorder_traversal(self, root.iter_names(), traversal_delegate).await + async_depth_first_postorder_traversal( + &QueryEnvironmentAsNodeLookup { env: self }, + root.iter_names(), + traversal_delegate, + visit, + ) + .await } async fn depth_limited_traversal( &self, root: &TargetSet, - delegate: &mut dyn AsyncTraversalDelegate, + delegate: impl AsyncChildVisitor, + visit: impl FnMut(Self::Target) -> anyhow::Result<()> + Send, depth: u32, ) -> anyhow::Result<()> { - async_depth_limited_traversal(self, root.iter_names(), delegate, depth).await + async_depth_limited_traversal( + &QueryEnvironmentAsNodeLookup { env: self }, + root.iter_names(), + delegate, + visit, + depth, + ) + .await } async fn allbuildfiles(&self, universe: &TargetSet) -> anyhow::Result { @@ -269,7 +283,7 @@ impl<'c> QueryEnvironment for UqueryEnvironment<'c> { .filter_map(|node| { for input in node.inputs() { if &input == path { - return Some(node.dupe()); + return Some(node.to_owned()); // this intentionally breaks out of the loop. We don't need to look at the // other inputs of this target, but it's possible for a single file to be owned by // multiple targets. @@ -301,13 +315,6 @@ impl<'c> QueryEnvironment for UqueryEnvironment<'c> { } } -#[async_trait] -impl<'a> AsyncNodeLookup for UqueryEnvironment<'a> { - async fn get(&self, label: &TargetLabel) -> anyhow::Result { - self.get_node(label).await - } -} - pub(crate) async fn allbuildfiles<'c, T: QueryTarget>( universe: &TargetSet, delegate: &'c dyn UqueryDelegate, @@ -375,12 +382,12 @@ pub(crate) async fn rbuildfiles<'c>( #[repr(transparent)] struct NodeRef(ImportPath); - impl NodeLabel for NodeRef {} + impl NodeKey for NodeRef {} impl LabeledNode for Node { - type NodeRef = NodeRef; + type Key = NodeRef; - fn node_ref(&self) -> &NodeRef { + fn node_key(&self) -> &NodeRef { NodeRef::ref_cast(self.import_path()) } } @@ -394,67 +401,65 @@ pub(crate) async fn rbuildfiles<'c>( } } - struct Delegate<'c> { - output_paths: Vec, - argset: &'c FileSet, - first_order_import_map: HashMap>, + let mut output_paths: Vec = Vec::new(); + + struct Delegate<'a> { + first_order_import_map: &'a HashMap>, } - #[async_trait] - impl AsyncTraversalDelegate for Delegate<'_> { - fn visit(&mut self, node: Node) -> anyhow::Result<()> { - let node_import = node.import_path(); - if self.argset.iter().contains(node_import.path()) { - self.output_paths.push(node_import.clone()); - } else { - let loads = self - .first_order_import_map - .get(node_import) - .expect("import path should exist"); - for load in loads.iter() { - for arg in self.argset.iter() { - if load.path() == arg { - self.output_paths.push(node_import.clone()); - return Ok(()); - } + let visit = |node: Node| { + let node_import = node.import_path(); + if argset.iter().contains(node_import.path()) { + output_paths.push(node_import.clone()); + } else { + let loads = first_order_import_map + .get(node_import) + .expect("import path should exist"); + for load in loads.iter() { + for arg in argset.iter() { + if load.path() == arg { + output_paths.push(node_import.clone()); + return Ok(()); } } } - Ok(()) } + Ok(()) + }; + + impl AsyncChildVisitor for Delegate<'_> { async fn for_each_child( - &mut self, + &self, node: &Node, - func: &mut dyn ChildVisitor, + mut func: impl ChildVisitor, ) -> anyhow::Result<()> { for import in self .first_order_import_map .get(node.import_path()) .expect("import path should exist") { - func.visit(NodeRef(import.clone()))?; + func.visit(&NodeRef(import.clone()))?; } Ok(()) } } let lookup = Lookup {}; - let mut delegate = Delegate { - output_paths: vec![], - argset, - first_order_import_map, + let delegate = Delegate { + first_order_import_map: &first_order_import_map, }; // step 5: do traversal, get all modified imports async_depth_first_postorder_traversal( &lookup, all_top_level_imports.iter().map(NodeRef::ref_cast), - &mut delegate, + delegate, + visit, ) .await?; let mut output_files = IndexSet::::new(); - for file in &delegate.output_paths { + for file in &output_paths { output_files.insert(FileNode(file.path().clone())); } @@ -592,12 +597,12 @@ pub(crate) async fn get_transitive_loads<'c>( #[repr(transparent)] struct NodeRef(ImportPath); - impl NodeLabel for NodeRef {} + impl NodeKey for NodeRef {} impl LabeledNode for Node { - type NodeRef = NodeRef; + type Key = NodeRef; - fn node_ref(&self) -> &NodeRef { + fn node_key(&self) -> &NodeRef { NodeRef::ref_cast(self.import_path()) } } @@ -611,43 +616,42 @@ pub(crate) async fn get_transitive_loads<'c>( } } + let mut imports: Vec = Vec::new(); + struct Delegate<'c> { - imports: Vec, delegate: &'c dyn UqueryDelegate, } - #[async_trait] - impl AsyncTraversalDelegate for Delegate<'_> { - fn visit(&mut self, target: Node) -> anyhow::Result<()> { - self.imports.push(target.import_path().clone()); - Ok(()) - } + let visit = |target: Node| { + imports.push(target.import_path().clone()); + Ok(()) + }; + impl AsyncChildVisitor for Delegate<'_> { async fn for_each_child( - &mut self, + &self, target: &Node, - func: &mut dyn ChildVisitor, + mut func: impl ChildVisitor, ) -> anyhow::Result<()> { for import in self .delegate .eval_module_imports(target.import_path()) .await? { - func.visit(NodeRef(import.clone()))?; + func.visit(&NodeRef(import))?; } Ok(()) } } - let mut traversal_delegate = Delegate { - imports: vec![], + let traversal_delegate = Delegate { delegate: delegate.dupe(), }; let lookup = Lookup {}; let import_nodes = top_level_imports.iter().map(NodeRef::ref_cast); - async_depth_first_postorder_traversal(&lookup, import_nodes, &mut traversal_delegate).await?; + async_depth_first_postorder_traversal(&lookup, import_nodes, traversal_delegate, visit).await?; - Ok(traversal_delegate.imports) + Ok(imports) } diff --git a/app/buck2_query_impls/src/uquery/evaluator.rs b/app/buck2_query_impls/src/uquery/evaluator.rs index 855159e08d792..c06817ca8b1a8 100644 --- a/app/buck2_query_impls/src/uquery/evaluator.rs +++ b/app/buck2_query_impls/src/uquery/evaluator.rs @@ -10,12 +10,14 @@ //! Implementation of the cli and query_* attr query language. use std::sync::Arc; +use buck2_common::events::HasEvents; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::fs::project_rel_path::ProjectRelativePath; -use buck2_core::target::label::TargetLabel; use buck2_node::nodes::unconfigured::TargetNode; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; use buck2_query::query::syntax::simple::functions::DefaultQueryFunctionsModule; use dice::DiceComputations; +use dupe::Dupe; use crate::analysis::evaluator::eval_query; use crate::dice::get_dice_query_delegate; @@ -23,42 +25,51 @@ use crate::dice::DiceQueryDelegate; use crate::uquery::environment::PreresolvedQueryLiterals; use crate::uquery::environment::UqueryEnvironment; -pub struct UqueryEvaluator<'c> { +pub(crate) struct UqueryEvaluator<'c> { dice_query_delegate: DiceQueryDelegate<'c>, functions: DefaultQueryFunctionsModule>, } impl UqueryEvaluator<'_> { - pub async fn eval_query( + pub(crate) async fn eval_query( &self, query: &str, query_args: &[String], ) -> anyhow::Result> { - eval_query(&self.functions, query, query_args, async move |literals| { - let resolved_literals = PreresolvedQueryLiterals::pre_resolve( - &**self.dice_query_delegate.query_data(), - &literals, - self.dice_query_delegate.ctx(), - ) - .await; - Ok(UqueryEnvironment::new( - &self.dice_query_delegate, - Arc::new(resolved_literals), - )) - }) + eval_query( + self.dice_query_delegate + .ctx() + .per_transaction_data() + .get_dispatcher() + .dupe(), + &self.functions, + query, + query_args, + async move |literals| { + let resolved_literals = PreresolvedQueryLiterals::pre_resolve( + &**self.dice_query_delegate.query_data(), + &literals, + self.dice_query_delegate.ctx(), + ) + .await; + Ok(UqueryEnvironment::new( + &self.dice_query_delegate, + Arc::new(resolved_literals), + )) + }, + ) .await } } /// Evaluates some query expression. TargetNodes are resolved via the interpreter from /// the provided DiceCtx. -pub async fn get_uquery_evaluator<'a, 'c: 'a>( +pub(crate) async fn get_uquery_evaluator<'a, 'c: 'a>( ctx: &'c DiceComputations, working_dir: &'a ProjectRelativePath, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, ) -> anyhow::Result> { - let dice_query_delegate = - get_dice_query_delegate(ctx, working_dir, global_target_platform).await?; + let dice_query_delegate = get_dice_query_delegate(ctx, working_dir, global_cfg_options).await?; let functions = DefaultQueryFunctionsModule::new(); Ok(UqueryEvaluator { diff --git a/app/buck2_query_impls/src/uquery/mod.rs b/app/buck2_query_impls/src/uquery/mod.rs index 9c43548eb0f28..6849c9eec9575 100644 --- a/app/buck2_query_impls/src/uquery/mod.rs +++ b/app/buck2_query_impls/src/uquery/mod.rs @@ -8,5 +8,5 @@ */ pub(crate) mod bxl; -pub mod environment; -pub mod evaluator; +pub(crate) mod environment; +pub(crate) mod evaluator; diff --git a/app/buck2_query_parser/src/lib.rs b/app/buck2_query_parser/src/lib.rs index 889f10a5e8e5a..b2c234ba070af 100644 --- a/app/buck2_query_parser/src/lib.rs +++ b/app/buck2_query_parser/src/lib.rs @@ -18,7 +18,6 @@ //! The grammar is something like: //! //! ```text -//! //! # note that set's args are space-separated, not comma-separated and so cannot be treated as a function //! EXPR ::= //! WORD @@ -43,9 +42,9 @@ //! INTEGER ::= "0" | ("1-9" "0-9"*) //! //! FUNCTION_NAME ::= "a-zA-Z_" "a-zA-Z0-9_" * -//! //! ``` +pub mod multi_query; pub mod placeholder; pub mod span; pub mod spanned; diff --git a/app/buck2_query_parser/src/multi_query.rs b/app/buck2_query_parser/src/multi_query.rs new file mode 100644 index 0000000000000..e9abbb0d6f769 --- /dev/null +++ b/app/buck2_query_parser/src/multi_query.rs @@ -0,0 +1,65 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use gazebo::prelude::SliceExt; + +use crate::placeholder::QUERY_PERCENT_S_PLACEHOLDER; + +#[derive(Debug, buck2_error::Error)] +enum EvalQueryError { + #[error("Query args supplied without any `%s` placeholder in the query, got args {}", .0.map(|x| format!("`{}`", x)).join(", "))] + ArgsWithoutPlaceholder(Vec), + #[error("Placeholder `%s` in query argument `{0}`")] + PlaceholderInPattern(String), +} + +pub struct MultiQueryItem { + pub arg: String, + pub query: String, +} + +/// Parsed query with optional `%s` placeholder and optional query args. +pub enum MaybeMultiQuery { + SingleQuery(String), + MultiQuery(Vec), +} + +impl MaybeMultiQuery { + pub fn parse( + query: &str, + args: impl IntoIterator>, + ) -> anyhow::Result { + if query.contains(QUERY_PERCENT_S_PLACEHOLDER) { + // We'd really like the query args to only be literals (file or target). + // If that didn't work, we'd really like query args to be well-formed expressions. + // Unfortunately Buck1 just substitutes in arbitrarily strings, where the query + // or query_args may not form anything remotely valid. + // We have to be backwards compatible :( + let queries = args + .into_iter() + .map(|arg| { + let arg = arg.as_ref().to_owned(); + if arg.contains(QUERY_PERCENT_S_PLACEHOLDER) { + return Err(EvalQueryError::PlaceholderInPattern(arg).into()); + } + let query = query.replace(QUERY_PERCENT_S_PLACEHOLDER, &arg); + Ok(MultiQueryItem { arg, query }) + }) + .collect::>()?; + Ok(MaybeMultiQuery::MultiQuery(queries)) + } else { + let args: Vec = args.into_iter().map(|q| q.as_ref().to_owned()).collect(); + if !args.is_empty() { + Err(EvalQueryError::ArgsWithoutPlaceholder(args).into()) + } else { + Ok(MaybeMultiQuery::SingleQuery(query.to_owned())) + } + } + } +} diff --git a/app/buck2_query_parser/src/placeholder.rs b/app/buck2_query_parser/src/placeholder.rs index 2dfa423e74ba9..98bd04a266855 100644 --- a/app/buck2_query_parser/src/placeholder.rs +++ b/app/buck2_query_parser/src/placeholder.rs @@ -7,7 +7,7 @@ * of this source tree. */ -/// Replace `%s` with remaining command line arguments which contain query literals. +/// Perform multiple queries with `%s` substituted with each of the arguments. pub const QUERY_PERCENT_S_PLACEHOLDER: &str = "%s"; -/// Replace `%Ss` with query literals read from files from remaining command line arguments. +/// Perform single query with `%Ss` substituted with `set(...)` of the arguments. pub const QUERY_PERCENT_SS_PLACEHOLDER: &str = "%Ss"; diff --git a/app/buck2_server/src/ctx.rs b/app/buck2_server/src/ctx.rs index 44b79749121cb..481950eb39e8e 100644 --- a/app/buck2_server/src/ctx.rs +++ b/app/buck2_server/src/ctx.rs @@ -267,12 +267,7 @@ impl<'a> ServerCommandContext<'a> { })); // Add argfiles read by client into IO tracing state. - if let Some(tracing_provider) = base_context - .daemon - .io - .as_any() - .downcast_ref::() - { + if let Some(tracing_provider) = TracingIoProvider::from_io(&*base_context.daemon.io) { let argfiles: anyhow::Result> = client_context .argfiles .iter() @@ -489,11 +484,11 @@ impl CellConfigLoader { truncate_container(self.config_overrides.iter().map(|o| o.to_string()), 200), ); } - return Ok::<(CellResolver, LegacyBuckConfigs, HashSet), anyhow::Error>(( + return buck2_error::Ok(( dice_ctx.get_cell_resolver().await?, dice_ctx.get_legacy_configs().await?, HashSet::new(), - )).map_err(buck2_error::Error::from); + )); } else { // If there is no previous command but the flag was set, then the flag is ignored, the command behaves as if there isn't the reuse config flag. warn!( @@ -842,6 +837,15 @@ impl<'a> ServerCommandContextTrait for ServerCommandContext<'a> { .to_string(), ); + metadata.insert( + "http_versions".to_owned(), + match self.base_context.daemon.http_client.http2() { + true => "1,2", + false => "1", + } + .to_owned(), + ); + Ok(metadata) } @@ -882,13 +886,7 @@ impl<'a> ServerCommandContextTrait for ServerCommandContext<'a> { self.cell_configs_loader.cells_and_configs(ctx).await?; // Add legacy config paths to I/O tracing (if enabled). - if let Some(tracing_provider) = self - .base_context - .daemon - .io - .as_any() - .downcast_ref::() - { + if let Some(tracing_provider) = TracingIoProvider::from_io(&*self.base_context.daemon.io) { tracing_provider.add_config_paths(&self.base_context.project_root, paths); } diff --git a/app/buck2_server/src/daemon/server.rs b/app/buck2_server/src/daemon/server.rs index e16a8d51f9a67..27a59c9d3e9b4 100644 --- a/app/buck2_server/src/daemon/server.rs +++ b/app/buck2_server/src/daemon/server.rs @@ -16,7 +16,6 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use std::task::Context; use std::task::Poll; -use std::thread; use std::time::Duration; use std::time::Instant; use std::time::SystemTime; @@ -63,6 +62,7 @@ use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; use buck2_server_ctx::streaming_request_handler::StreamingRequestHandler; use buck2_server_ctx::test_command::TEST_COMMAND; use buck2_server_starlark_debug::run::run_dap_server_command; +use buck2_util::threads::thread_spawn; use dice::DetectCycles; use dice::Dice; use dice::WhichDice; @@ -209,10 +209,7 @@ impl Interceptor for BuckCheckAuthTokenInterceptor { Some(token) => token, None => return Err(Status::unauthenticated("missing auth token")), }; - if !constant_time_eq::constant_time_eq( - token.as_bytes(), - self.auth_token.as_str().as_bytes(), - ) { + if !constant_time_eq::constant_time_eq(token.as_bytes(), self.auth_token.as_bytes()) { return Err(Status::unauthenticated("invalid auth token")); } @@ -668,11 +665,9 @@ where // We run the event consumer on new non-tokio thread to avoid the consumer task from getting stuck behind // another tokio task in its lifo task slot. See T96012305 and https://github.com/tokio-rs/tokio/issues/4323 for more // information. - let merge_task = thread::Builder::new() - .name("pump-events".to_owned()) - .spawn(move || { - pump_events(events, state, output_send); - }); + let merge_task = thread_spawn("pump-events", move || { + pump_events(events, state, output_send); + }); if let Err(e) = merge_task { return error_to_response_stream( anyhow::Error::new(e).context("failed to spawn pump-events"), @@ -798,7 +793,7 @@ impl DaemonApi for BuckdServer { let extra_constraints = daemon_state.data().as_ref().ok().map(|state| { buck2_cli_proto::ExtraDaemonConstraints { - trace_io_enabled: state.io.as_any().is::(), + trace_io_enabled: TracingIoProvider::from_io(&*state.io).is_some(), materializer_state_identity: state .materializer_state_identity .as_ref() @@ -828,6 +823,11 @@ impl DaemonApi for BuckdServer { .as_ref() .ok() .map(|state| state.http_client.supports_vpnless()), + http2: daemon_state + .data() + .as_ref() + .ok() + .map(|state| state.http_client.http2()), ..Default::default() }; Ok(base) @@ -1082,9 +1082,14 @@ impl DaemonApi for BuckdServer { async fn unstable_crash( &self, - _req: Request, - ) -> Result, Status> { - panic!("explicitly requested panic (via unstable_crash)"); + req: Request, + ) -> Result, Status> { + self.oneshot(req, DefaultCommandOptions, move |_req| async move { + panic!("explicitly requested panic (via unstable_crash)"); + #[allow(unreachable_code)] + Ok(GenericResponse {}) + }) + .await } async fn segfault( diff --git a/app/buck2_server/src/daemon/state.rs b/app/buck2_server/src/daemon/state.rs index defee67a60548..61d4468f51ee1 100644 --- a/app/buck2_server/src/daemon/state.rs +++ b/app/buck2_server/src/daemon/state.rs @@ -815,6 +815,7 @@ fn http_client_from_startup_config( HttpClientBuilder::internal(config.allow_vpnless)? }; builder.with_max_redirects(config.http.max_redirects.unwrap_or(DEFAULT_MAX_REDIRECTS)); + builder.with_http2(config.http.http2); match config.http.connect_timeout() { Timeout::Value(d) => { builder.with_connect_timeout(Some(d)); diff --git a/app/buck2_server/src/dice_tracker.rs b/app/buck2_server/src/dice_tracker.rs index 1d59961e2a27d..358b774e86ba2 100644 --- a/app/buck2_server/src/dice_tracker.rs +++ b/app/buck2_server/src/dice_tracker.rs @@ -14,6 +14,7 @@ use allocative::Allocative; use buck2_data::*; use buck2_events::dispatch::with_dispatcher_async; use buck2_events::dispatch::EventDispatcher; +use buck2_util::threads::thread_spawn; use dice::DiceEvent; use dice::DiceEventListener; use dupe::Dupe; @@ -42,7 +43,7 @@ impl BuckDiceTracker { pub fn new(events: EventDispatcher) -> Self { let (event_forwarder, receiver) = mpsc::unbounded(); - std::thread::spawn(move || { + thread_spawn("buck2-dice-tracker", move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -51,7 +52,8 @@ impl BuckDiceTracker { events.dupe(), Self::run_task(events, receiver), )) - }); + }) + .unwrap(); Self { event_forwarder } } diff --git a/app/buck2_server/src/file_status.rs b/app/buck2_server/src/file_status.rs index 22943972ac6bf..27d180dce758b 100644 --- a/app/buck2_server/src/file_status.rs +++ b/app/buck2_server/src/file_status.rs @@ -60,7 +60,7 @@ struct FileStatusResult<'a> { /// Number of ones that were bad bad: usize, /// Whether to write matches - verbose: bool, + show_matches: bool, stdout: StdoutPartialOutput<'a>, } @@ -86,7 +86,7 @@ impl FileStatusResult<'_> { kind, path, fs, dice, )?; self.bad += 1; - } else if self.verbose { + } else if self.show_matches { writeln!(self.stdout, "Match: {} at {}: {}", kind, path, fs)?; } @@ -132,7 +132,7 @@ impl ServerCommandTemplate for FileStatusServerCommand { let mut result = FileStatusResult { checked: 0, bad: 0, - verbose: self.req.verbose, + show_matches: self.req.show_matches, stdout, }; diff --git a/app/buck2_server/src/lsp.rs b/app/buck2_server/src/lsp.rs index 9ca8583434930..b0b8b543d2b9d 100644 --- a/app/buck2_server/src/lsp.rs +++ b/app/buck2_server/src/lsp.rs @@ -925,7 +925,7 @@ fn handle_outgoing_lsp_message( } #[cfg(test)] -mod test { +mod tests { use lsp_types::Url; use maplit::hashmap; use starlark::docs::Doc; diff --git a/app/buck2_server/src/profile.rs b/app/buck2_server/src/profile.rs index b4db7f8a3a6a0..804c3bbeea60c 100644 --- a/app/buck2_server/src/profile.rs +++ b/app/buck2_server/src/profile.rs @@ -19,6 +19,7 @@ use buck2_cli_proto::target_profile::Action; use buck2_cli_proto::ClientContext; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::file_ops::HasFileOps; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::pattern::resolve::resolve_target_patterns; use buck2_core::cells::build_file_cell::BuildFileCell; use buck2_core::fs::paths::abs_path::AbsPath; @@ -38,8 +39,8 @@ use buck2_profile::starlark_profiler_configuration_from_request; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; @@ -50,7 +51,7 @@ async fn generate_profile_analysis( ctx: DiceTransaction, package: PackageLabel, spec: PackageSpec, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, profile_mode: &StarlarkProfilerConfiguration, ) -> anyhow::Result> { let (target, TargetPatternExtra) = match spec { @@ -62,7 +63,7 @@ async fn generate_profile_analysis( let label = TargetLabel::new(package.dupe(), target.as_ref()); let configured_target = ctx - .get_configured_target(&label, global_target_platform.as_ref()) + .get_configured_target(&label, &global_cfg_options) .await?; match profile_mode { @@ -191,8 +192,8 @@ async fn generate_profile( ) -> anyhow::Result> { let cells = ctx.get_cell_resolver().await?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let parsed_patterns = parse_patterns_from_cli_args::( &mut ctx, @@ -207,8 +208,7 @@ async fn generate_profile( Action::Analysis => { let (package, spec) = one(resolved.specs) .context("Error: profiling analysis requires exactly one target pattern")?; - generate_profile_analysis(ctx, package, spec, global_target_platform, profile_mode) - .await + generate_profile_analysis(ctx, package, spec, global_cfg_options, profile_mode).await } Action::Loading => { let ctx = &ctx; diff --git a/app/buck2_server/src/trace_io.rs b/app/buck2_server/src/trace_io.rs index ee7412f950f53..6b45c55c03de0 100644 --- a/app/buck2_server/src/trace_io.rs +++ b/app/buck2_server/src/trace_io.rs @@ -27,12 +27,7 @@ pub(crate) async fn trace_io_command( data: Some(buck2_data::TraceIoCommandStart {}.into()), }; span_async(start_event, async move { - let tracing_provider = &context - .base_context - .daemon - .io - .as_any() - .downcast_ref::(); + let tracing_provider = TracingIoProvider::from_io(&*context.base_context.daemon.io); let respond_with_trace = matches!( req.read_state, Some(trace_io_request::ReadIoTracingState { with_trace: true }) diff --git a/app/buck2_server_commands/BUCK b/app/buck2_server_commands/BUCK index b76eb6e73f32e..7e88c8ec63f01 100644 --- a/app/buck2_server_commands/BUCK +++ b/app/buck2_server_commands/BUCK @@ -12,7 +12,6 @@ rust_library( "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:blake3", "fbsource//third-party/rust:chrono", - "fbsource//third-party/rust:derivative", "fbsource//third-party/rust:derive_more", "fbsource//third-party/rust:futures", "fbsource//third-party/rust:indent_write", @@ -34,7 +33,6 @@ rust_library( "//buck2/app/buck2_core:buck2_core", "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_error:buck2_error", - "//buck2/app/buck2_event_observer:buck2_event_observer", "//buck2/app/buck2_events:buck2_events", "//buck2/app/buck2_execute:buck2_execute", "//buck2/app/buck2_futures:buck2_futures", @@ -44,7 +42,6 @@ rust_library( "//buck2/app/buck2_query:buck2_query", "//buck2/app/buck2_server_ctx:buck2_server_ctx", "//buck2/app/buck2_util:buck2_util", - "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/dice/dice:dice", "//buck2/gazebo/dupe:dupe", "//buck2/gazebo/gazebo:gazebo", diff --git a/app/buck2_server_commands/Cargo.toml b/app/buck2_server_commands/Cargo.toml index dd828838bb480..3ddfc8231f7a9 100644 --- a/app/buck2_server_commands/Cargo.toml +++ b/app/buck2_server_commands/Cargo.toml @@ -12,7 +12,6 @@ async-recursion = { workspace = true } async-trait = { workspace = true } blake3 = { workspace = true } chrono = { workspace = true } -derivative = { workspace = true } derive_more = { workspace = true } futures = { workspace = true } indent_write = { workspace = true } @@ -40,7 +39,6 @@ buck2_common = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } buck2_error = { workspace = true } -buck2_event_observer = { workspace = true } buck2_events = { workspace = true } buck2_execute = { workspace = true } buck2_futures = { workspace = true } @@ -50,4 +48,3 @@ buck2_node = { workspace = true } buck2_query = { workspace = true } buck2_server_ctx = { workspace = true } buck2_util = { workspace = true } -buck2_wrapper_common = { workspace = true } diff --git a/app/buck2_server_commands/src/commands/build/mod.rs b/app/buck2_server_commands/src/commands/build/mod.rs index 55eed6fec2385..6411a2127b7fa 100644 --- a/app/buck2_server_commands/src/commands/build/mod.rs +++ b/app/buck2_server_commands/src/commands/build/mod.rs @@ -21,6 +21,7 @@ use buck2_artifact::artifact::artifact_dump::FileInfo; use buck2_artifact::artifact::artifact_dump::SymlinkInfo; use buck2_build_api::actions::artifact::get_artifact_fs::GetArtifactFs; use buck2_build_api::build; +use buck2_build_api::build::build_report::BuildReportCollector; use buck2_build_api::build::BuildEvent; use buck2_build_api::build::BuildTargetResult; use buck2_build_api::build::ConfiguredBuildEvent; @@ -37,6 +38,7 @@ use buck2_cli_proto::CommonBuildOptions; use buck2_cli_proto::HasClientContext; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::file_ops::HasFileOps; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::legacy_configs::dice::HasLegacyConfigs; use buck2_common::pattern::resolve::resolve_target_patterns; use buck2_common::pattern::resolve::ResolvedPattern; @@ -66,8 +68,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceComputations; @@ -82,14 +84,11 @@ use itertools::Itertools; use serde::ser::SerializeSeq; use serde::ser::Serializer; -use crate::commands::build::build_report::BuildReportCollector; use crate::commands::build::result_report::ResultReporter; use crate::commands::build::result_report::ResultReporterOptions; use crate::commands::build::unhashed_outputs::create_unhashed_outputs; #[allow(unused)] -mod action_error; -mod build_report; mod result_report; mod unhashed_outputs; @@ -141,7 +140,7 @@ impl ServerCommandTemplate for BuildServerCommand { enum TargetResolutionConfig { /// Resolve using target platform. - Default(Option), + Default(GlobalCfgOptions), /// Resolve in the universe. Universe(CqueryUniverse), } @@ -218,8 +217,8 @@ async fn build( let cell_resolver = ctx.get_cell_resolver().await?; let client_ctx = request.client_context()?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let parsed_patterns: Vec> = parse_patterns_from_cli_args(&mut ctx, &request.target_patterns, cwd).await?; @@ -229,12 +228,12 @@ async fn build( resolve_target_patterns(&cell_resolver, &parsed_patterns, &ctx.file_ops()).await?; let target_resolution_config: TargetResolutionConfig = if request.target_universe.is_empty() { - TargetResolutionConfig::Default(global_target_platform) + TargetResolutionConfig::Default(global_cfg_options) } else { TargetResolutionConfig::Universe( QUERY_FRONTEND .get()? - .universe_from_literals(&ctx, cwd, &request.target_universe, global_target_platform) + .universe_from_literals(&ctx, cwd, &request.target_universe, global_cfg_options) .await?, ) }; @@ -316,7 +315,9 @@ async fn process_build_result( ) .await? .unwrap_or(false), - &build_result, + build_opts.unstable_include_failures_build_report, + &build_result.configured, + &build_result.other_errors, )) } else { None @@ -413,14 +414,14 @@ async fn build_targets( want_configured_graph_size: bool, ) -> anyhow::Result { let stream = match target_resolution_config { - TargetResolutionConfig::Default(global_target_platform) => { + TargetResolutionConfig::Default(global_cfg_options) => { let spec = spec.convert_pattern().context( "Cannot build with explicit configurations when universe is not specified", )?; build_targets_with_global_target_platform( ctx, spec, - global_target_platform, + global_cfg_options, build_providers, materialization_context, missing_target_behavior, @@ -479,7 +480,7 @@ fn build_targets_in_universe<'a>( fn build_targets_with_global_target_platform<'a>( ctx: &'a DiceComputations, spec: ResolvedPattern, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, build_providers: Arc, materialization_context: &'a MaterializationContext, missing_target_behavior: MissingTargetBehavior, @@ -491,7 +492,7 @@ fn build_targets_with_global_target_platform<'a>( ctx, spec, package, - global_target_platform.dupe(), + global_cfg_options.dupe(), build_providers.dupe(), materialization_context, missing_target_behavior, @@ -507,7 +508,7 @@ fn build_targets_with_global_target_platform<'a>( struct TargetBuildSpec { target: TargetNode, providers: ProvidersName, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, // Indicates whether this target was explicitly requested or not. If it's the result // of something like `//foo/...` we can skip it (for example if it's incompatible with // the target platform). @@ -538,7 +539,7 @@ async fn build_targets_for_spec<'a>( ctx: &'a DiceComputations, spec: PackageSpec, package: PackageLabel, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, build_providers: Arc, materialization_context: &'a MaterializationContext, missing_target_behavior: MissingTargetBehavior, @@ -606,7 +607,7 @@ async fn build_targets_for_spec<'a>( .map(|((_target_name, extra), target)| TargetBuildSpec { target, providers: extra.providers, - global_target_platform: global_target_platform.dupe(), + global_cfg_options: global_cfg_options.dupe(), skippable, want_configured_graph_size, }) @@ -642,7 +643,7 @@ async fn build_target<'a>( ) -> impl Stream + 'a { let providers_label = ProvidersLabel::new(spec.target.label().dupe(), spec.providers); let providers_label = match ctx - .get_configured_provider_label(&providers_label, spec.global_target_platform.as_ref()) + .get_configured_provider_label(&providers_label, &spec.global_cfg_options) .await { Ok(l) => l, diff --git a/app/buck2_server_commands/src/commands/build/unhashed_outputs.rs b/app/buck2_server_commands/src/commands/build/unhashed_outputs.rs index 19a97fb5d307d..5c0fe787e98a3 100644 --- a/app/buck2_server_commands/src/commands/build/unhashed_outputs.rs +++ b/app/buck2_server_commands/src/commands/build/unhashed_outputs.rs @@ -149,7 +149,7 @@ fn iter_reverse_ancestors<'a>( } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_server_commands/src/commands/ctargets/mod.rs b/app/buck2_server_commands/src/commands/ctargets/mod.rs index 2d89fbc1ccb32..2ea911191f941 100644 --- a/app/buck2_server_commands/src/commands/ctargets/mod.rs +++ b/app/buck2_server_commands/src/commands/ctargets/mod.rs @@ -20,8 +20,8 @@ use buck2_node::load_patterns::MissingTargetBehavior; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; @@ -78,15 +78,15 @@ impl ServerCommandTemplate for ConfiguredTargetsServerCommand { let target_call_stacks = client_ctx.target_call_stacks; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let skip_missing_targets = MissingTargetBehavior::from_skip(self.req.skip_missing_targets); let compatible_targets = load_compatible_patterns( &ctx, parsed_patterns, - global_target_platform, + &global_cfg_options, skip_missing_targets, ) .await?; diff --git a/app/buck2_server_commands/src/commands/debug_eval.rs b/app/buck2_server_commands/src/commands/debug_eval.rs index fcef16b963c08..cb0db7a30b330 100644 --- a/app/buck2_server_commands/src/commands/debug_eval.rs +++ b/app/buck2_server_commands/src/commands/debug_eval.rs @@ -34,6 +34,7 @@ pub(crate) async fn debug_eval_command( ) -> anyhow::Result { context .with_dice_ctx(|server_ctx, ctx| async move { + let ctx = &ctx; let cell_resolver = ctx.get_cell_resolver().await?; let current_cell_path = cell_resolver.get_cell_path(server_ctx.working_dir())?; let mut loads = Vec::new(); @@ -52,10 +53,7 @@ pub(crate) async fn debug_eval_command( } else { return Err(DebugEvalError::InvalidImportPath(path).into()); }; - loads.push(async { - let import_path = import_path; - ctx.get_loaded_module(import_path.borrow()).await - }); + loads.push(async move { ctx.get_loaded_module(import_path.borrow()).await }); } // Catch errors, ignore results. diff --git a/app/buck2_server_commands/src/commands/install.rs b/app/buck2_server_commands/src/commands/install.rs index 764e6dd228d8b..7169ea725c4c7 100644 --- a/app/buck2_server_commands/src/commands/install.rs +++ b/app/buck2_server_commands/src/commands/install.rs @@ -66,8 +66,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use buck2_util::process::background_command; @@ -181,8 +181,8 @@ async fn install( let cell_resolver = ctx.get_cell_resolver().await?; let client_ctx = request.client_context()?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let materializations = MaterializationContext::force_materializations(); let materializations = &materializations; // Don't move this below. @@ -227,7 +227,7 @@ async fn install( for (target_name, providers) in targets { let label = providers.into_providers_label(package.dupe(), target_name.as_ref()); let providers_label = ctx - .get_configured_provider_label(&label, global_target_platform.dupe().as_ref()) + .get_configured_provider_label(&label, &global_cfg_options) .await?; let frozen_providers = ctx .get_providers(&providers_label) diff --git a/app/buck2_server_commands/src/commands/query/aquery.rs b/app/buck2_server_commands/src/commands/query/aquery.rs index 17954373c15fe..3a6f1d156869a 100644 --- a/app/buck2_server_commands/src/commands/query/aquery.rs +++ b/app/buck2_server_commands/src/commands/query/aquery.rs @@ -11,18 +11,38 @@ use std::io::Write; use anyhow::Context; use async_trait::async_trait; +use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::query::oneshot::QUERY_FRONTEND; use buck2_common::dice::cells::HasCellResolver; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; use crate::commands::query::printer::QueryResultPrinter; use crate::commands::query::printer::ShouldPrintProviders; +use crate::commands::query::query_target_ext::QueryCommandTarget; + +impl QueryCommandTarget for ActionQueryNode { + fn call_stack(&self) -> Option { + None + } + + fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { + format!("{:#}", attr) + } + + fn attr_serialize( + &self, + attr: &Self::Attr<'_>, + serializer: S, + ) -> Result { + serde::Serialize::serialize(attr, serializer) + } +} pub(crate) async fn aquery_command( ctx: &dyn ServerCommandContextTrait, @@ -87,8 +107,8 @@ async fn aquery( let client_ctx = context .as_ref() .context("No client context (internal error)")?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let query_result = QUERY_FRONTEND .get()? @@ -97,7 +117,7 @@ async fn aquery( server_ctx.working_dir(), query, query_args, - global_target_platform, + global_cfg_options, ) .await?; diff --git a/app/buck2_server_commands/src/commands/query/cquery.rs b/app/buck2_server_commands/src/commands/query/cquery.rs index 84d9c4a4b0538..f4ff46d7cdda3 100644 --- a/app/buck2_server_commands/src/commands/query/cquery.rs +++ b/app/buck2_server_commands/src/commands/query/cquery.rs @@ -21,11 +21,14 @@ use buck2_common::dice::cells::HasCellResolver; use buck2_core::configuration::compatibility::MaybeCompatible; use buck2_core::provider::label::ConfiguredProvidersLabel; use buck2_core::provider::label::ProvidersName; +use buck2_node::attrs::display::AttrDisplayWithContextExt; +use buck2_node::attrs::fmt_context::AttrFmtContext; +use buck2_node::attrs::serialize::AttrSerializeWithContext; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use buck2_util::truncate::truncate; @@ -36,6 +39,35 @@ use dupe::Dupe; use crate::commands::query::printer::ProviderLookUp; use crate::commands::query::printer::QueryResultPrinter; use crate::commands::query::printer::ShouldPrintProviders; +use crate::commands::query::query_target_ext::QueryCommandTarget; + +impl QueryCommandTarget for ConfiguredTargetNode { + fn call_stack(&self) -> Option { + ConfiguredTargetNode::call_stack(self) + } + + fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { + format!( + "{:#}", + attr.as_display(&AttrFmtContext { + package: Some(self.label().pkg().dupe()), + }) + ) + } + + fn attr_serialize( + &self, + attr: &Self::Attr<'_>, + serializer: S, + ) -> Result { + attr.serialize_with_ctx( + &AttrFmtContext { + package: Some(self.label().pkg().dupe()), + }, + serializer, + ) + } +} pub(crate) async fn cquery_command( ctx: &dyn ServerCommandContextTrait, @@ -118,8 +150,8 @@ async fn cquery( let target_call_stacks = client_ctx.target_call_stacks; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let owner_behavior = match correct_owner { true => CqueryOwnerBehavior::Correct, @@ -134,7 +166,7 @@ async fn cquery( owner_behavior, query, query_args, - global_target_platform, + global_cfg_options, target_universe, ) .await?; diff --git a/app/buck2_server_commands/src/commands/query/mod.rs b/app/buck2_server_commands/src/commands/query/mod.rs index aa07455e4c2d1..4b627d8b69216 100644 --- a/app/buck2_server_commands/src/commands/query/mod.rs +++ b/app/buck2_server_commands/src/commands/query/mod.rs @@ -10,6 +10,7 @@ pub mod aquery; pub mod cquery; pub mod printer; +pub(crate) mod query_target_ext; pub mod uquery; #[derive(Debug, buck2_error::Error)] diff --git a/app/buck2_server_commands/src/commands/query/printer.rs b/app/buck2_server_commands/src/commands/query/printer.rs index 102f258f8c75d..c6a1d6d31edcc 100644 --- a/app/buck2_server_commands/src/commands/query/printer.rs +++ b/app/buck2_server_commands/src/commands/query/printer.rs @@ -38,6 +38,7 @@ use serde::ser::SerializeSeq; use serde::Serialize; use serde::Serializer; +use crate::commands::query::query_target_ext::QueryCommandTarget; use crate::commands::query::QueryCommandError; use crate::dot::targets::DotTargetGraph; use crate::dot::Dot; @@ -56,7 +57,7 @@ pub trait ProviderLookUp: Send + Sync { } #[derive(Debug)] -pub struct QueryResultPrinter<'a> { +pub(crate) struct QueryResultPrinter<'a> { resolver: &'a CellResolver, attributes: Option, output_format: QueryOutputFormat, @@ -93,13 +94,13 @@ struct PrintableQueryTarget<'a, T: QueryTarget> { impl<'a, T: QueryTarget> PrintableQueryTarget<'a, T> { fn label(&self) -> String { - self.value.node_ref().to_string() + self.value.node_key().to_string() } } -impl<'a, T: QueryTarget> Display for PrintableQueryTarget<'a, T> { +impl<'a, T: QueryCommandTarget> Display for PrintableQueryTarget<'a, T> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.value.node_ref())?; + write!(f, "{}", self.value.node_key())?; if self.target_call_stacks || self.providers.is_some() { writeln!(f)?; @@ -129,7 +130,7 @@ impl<'a, T: QueryTarget> Display for PrintableQueryTarget<'a, T> { } } -impl<'a, T: QueryTarget> Serialize for PrintableQueryTarget<'a, T> { +impl<'a, T: QueryCommandTarget> Serialize for PrintableQueryTarget<'a, T> { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -139,12 +140,12 @@ impl<'a, T: QueryTarget> Serialize for PrintableQueryTarget<'a, T> { QueryTargets::for_all_attrs(self.value, |attr_name, attr_value| { if let Some(attr_regex) = self.attributes { if attr_regex.is_match(attr_name) { - struct AttrValueSerialize<'a, 'b, T: QueryTarget> { + struct AttrValueSerialize<'a, 'b, T: QueryCommandTarget> { target: &'a T, attr: &'a T::Attr<'b>, } - impl<'a, 'b, T: QueryTarget> Serialize for AttrValueSerialize<'a, 'b, T> { + impl<'a, 'b, T: QueryCommandTarget> Serialize for AttrValueSerialize<'a, 'b, T> { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -177,7 +178,7 @@ impl<'a, T: QueryTarget> Serialize for PrintableQueryTarget<'a, T> { } } -impl<'a, T: QueryTarget> Serialize for TargetSetJsonPrinter<'a, T> { +impl<'a, T: QueryCommandTarget> Serialize for TargetSetJsonPrinter<'a, T> { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -253,7 +254,7 @@ impl<'a> QueryResultPrinter<'a> { }) } - pub async fn print_multi_output<'b, T: QueryTarget, W: std::io::Write>( + pub async fn print_multi_output<'b, T: QueryCommandTarget, W: std::io::Write>( &self, mut output: W, multi_result: MultiQueryResult, @@ -319,7 +320,7 @@ impl<'a> QueryResultPrinter<'a> { } } - pub async fn print_single_output<'b, T: QueryTarget, W: std::io::Write>( + pub async fn print_single_output<'b, T: QueryCommandTarget, W: std::io::Write>( &self, mut output: W, result: QueryEvaluationValue, diff --git a/app/buck2_server_commands/src/commands/query/query_target_ext.rs b/app/buck2_server_commands/src/commands/query/query_target_ext.rs new file mode 100644 index 0000000000000..44dc586167ff0 --- /dev/null +++ b/app/buck2_server_commands/src/commands/query/query_target_ext.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use buck2_query::query::environment::QueryTarget; + +/// Extensions of `QueryTarget` needed in query commands. +pub(crate) trait QueryCommandTarget: QueryTarget { + fn call_stack(&self) -> Option; + + fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String; + + fn attr_serialize( + &self, + attr: &Self::Attr<'_>, + serializer: S, + ) -> Result; +} diff --git a/app/buck2_server_commands/src/commands/query/uquery.rs b/app/buck2_server_commands/src/commands/query/uquery.rs index 974a6fb13ee7d..b8a9b0627b73f 100644 --- a/app/buck2_server_commands/src/commands/query/uquery.rs +++ b/app/buck2_server_commands/src/commands/query/uquery.rs @@ -15,16 +15,51 @@ use buck2_build_api::query::oneshot::QUERY_FRONTEND; use buck2_cli_proto::UqueryRequest; use buck2_cli_proto::UqueryResponse; use buck2_common::dice::cells::HasCellResolver; +use buck2_node::attrs::display::AttrDisplayWithContextExt; +use buck2_node::attrs::fmt_context::AttrFmtContext; +use buck2_node::attrs::serialize::AttrSerializeWithContext; +use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeData; use buck2_query::query::syntax::simple::eval::values::QueryEvaluationResult; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; -use buck2_server_ctx::pattern::target_platform_from_client_context; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; +use dupe::Dupe; use crate::commands::query::printer::QueryResultPrinter; use crate::commands::query::printer::ShouldPrintProviders; +use crate::commands::query::query_target_ext::QueryCommandTarget; + +impl QueryCommandTarget for TargetNode { + fn call_stack(&self) -> Option { + TargetNodeData::call_stack(self) + } + + fn attr_to_string_alternate(&self, attr: &Self::Attr<'_>) -> String { + format!( + "{:#}", + attr.as_display(&AttrFmtContext { + package: Some(self.label().pkg().dupe()), + }) + ) + } + + fn attr_serialize( + &self, + attr: &Self::Attr<'_>, + serializer: S, + ) -> Result { + attr.serialize_with_ctx( + &AttrFmtContext { + package: Some(self.label().pkg().dupe()), + }, + serializer, + ) + } +} pub(crate) async fn uquery_command( ctx: &dyn ServerCommandContextTrait, @@ -91,8 +126,8 @@ async fn uquery( let target_call_stacks = client_ctx.target_call_stacks; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let query_result = QUERY_FRONTEND .get()? @@ -101,7 +136,7 @@ async fn uquery( server_ctx.working_dir(), query, query_args, - global_target_platform, + global_cfg_options, ) .await?; diff --git a/app/buck2_server_commands/src/commands/targets/default.rs b/app/buck2_server_commands/src/commands/targets/default.rs index 31d1fee81e2ce..6084edb161c94 100644 --- a/app/buck2_server_commands/src/commands/targets/default.rs +++ b/app/buck2_server_commands/src/commands/targets/default.rs @@ -17,12 +17,12 @@ use buck2_cli_proto::targets_request; use buck2_cli_proto::targets_request::TargetHashFileMode; use buck2_cli_proto::targets_request::TargetHashGraphType; use buck2_cli_proto::TargetsResponse; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::CellResolver; use buck2_core::fs::paths::abs_path::AbsPath; use buck2_core::fs::project::ProjectRoot; use buck2_core::pattern::pattern_type::TargetPatternExtra; use buck2_core::pattern::ParsedPattern; -use buck2_core::target::label::TargetLabel; use buck2_node::load_patterns::load_patterns; use buck2_node::load_patterns::MissingTargetBehavior; use buck2_node::nodes::configured::ConfiguredTargetNode; @@ -37,7 +37,6 @@ use dupe::OptionDupedExt; use crate::commands::targets::fmt::Stats; use crate::commands::targets::fmt::TargetFormatter; use crate::commands::targets::fmt::TargetInfo; -use crate::commands::targets::mk_error; use crate::target_hash::TargetHashes; use crate::target_hash::TargetHashesFileMode; @@ -87,7 +86,7 @@ pub(crate) async fn targets_batch( dice: DiceTransaction, formatter: &dyn TargetFormatter, parsed_patterns: Vec>, - target_platform: Option, + global_cfg_options: &GlobalCfgOptions, hash_options: TargetHashOptions, keep_going: bool, ) -> anyhow::Result { @@ -99,7 +98,7 @@ pub(crate) async fn targets_batch( dice.dupe(), ConfiguredTargetNodeLookup(&dice), results.iter_loaded_targets_by_package().collect(), - target_platform, + global_cfg_options, hash_options.file_mode, hash_options.fast_hash, hash_options.recursive, @@ -111,7 +110,7 @@ pub(crate) async fn targets_batch( dice.dupe(), TargetNodeLookup(&dice), results.iter_loaded_targets_by_package().collect(), - target_platform, + global_cfg_options, hash_options.file_mode, hash_options.fast_hash, hash_options.recursive, @@ -151,14 +150,14 @@ pub(crate) async fn targets_batch( } } Err(e) => { - stats.errors += 1; + stats.add_error(e); let mut stderr = String::new(); if needs_separator { formatter.separator(&mut buffer); } needs_separator = true; - formatter.package_error(package.dupe(), &e.dupe().into(), &mut buffer, &mut stderr); + formatter.package_error(package.dupe(), e, &mut buffer, &mut stderr); server_ctx.stderr()?.write_all(stderr.as_bytes())?; @@ -169,8 +168,8 @@ pub(crate) async fn targets_batch( } } formatter.end(&stats, &mut buffer); - if !keep_going && stats.errors != 0 { - Err(mk_error(stats.errors)) + if !keep_going && let Some(e) = stats.to_error() { + Err(e) } else { Ok(TargetsResponse { error_count: stats.errors, diff --git a/app/buck2_server_commands/src/commands/targets/fmt.rs b/app/buck2_server_commands/src/commands/targets/fmt.rs index d47f54047f038..d9aa04171f310 100644 --- a/app/buck2_server_commands/src/commands/targets/fmt.rs +++ b/app/buck2_server_commands/src/commands/targets/fmt.rs @@ -7,10 +7,10 @@ * of this source tree. */ +use std::collections::BTreeSet; use std::fmt::Write; use std::sync::Arc; -use anyhow::Context; use buck2_cli_proto::targets_request; use buck2_cli_proto::targets_request::OutputFormat; use buck2_cli_proto::targets_request::TargetHashGraphType; @@ -19,6 +19,7 @@ use buck2_cli_proto::TargetsRequest; use buck2_core::bzl::ImportPath; use buck2_core::cells::cell_path::CellPath; use buck2_core::package::PackageLabel; +use buck2_error::Context; use buck2_node::attrs::hacks::value_to_json; use buck2_node::attrs::inspect_options::AttrInspectOptions; use buck2_node::nodes::attributes::DEPS; @@ -29,7 +30,7 @@ use buck2_node::nodes::attributes::PACKAGE_VALUES; use buck2_node::nodes::attributes::TARGET_CALL_STACK; use buck2_node::nodes::attributes::TARGET_HASH; use buck2_node::nodes::attributes::TYPE; -use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeRef; use buck2_node::super_package::SuperPackage; use buck2_util::indent::indent; use gazebo::prelude::SliceExt; @@ -47,12 +48,16 @@ enum FormatterError { } pub(crate) struct TargetInfo<'a> { - pub(crate) node: &'a TargetNode, + pub(crate) node: TargetNodeRef<'a>, pub(crate) target_hash: Option, pub(crate) super_package: &'a SuperPackage, } -fn package_error_to_stderr(package: &PackageLabel, error: &anyhow::Error, stderr: &mut String) { +fn package_error_to_stderr( + package: &PackageLabel, + error: &buck2_error::Error, + stderr: &mut String, +) { writeln!(stderr, "Error parsing {package}\n{error:?}").unwrap(); } @@ -74,7 +79,7 @@ pub(crate) trait TargetFormatter: Send + Sync { fn package_error( &self, package: PackageLabel, - error: &anyhow::Error, + error: &buck2_error::Error, stdout: &mut String, stderr: &mut String, ) { @@ -294,7 +299,7 @@ impl TargetFormatter for JsonFormat { fn package_error( &self, package: PackageLabel, - error: &anyhow::Error, + error: &buck2_error::Error, stdout: &mut String, stderr: &mut String, ) { @@ -323,6 +328,8 @@ impl TargetFormatter for JsonFormat { #[derive(Debug, Default)] pub(crate) struct Stats { pub(crate) errors: u64, + error_tags: BTreeSet, + error_category: Option, pub(crate) success: u64, pub(crate) targets: u64, } @@ -333,6 +340,39 @@ impl Stats { self.success += stats.success; self.targets += stats.targets; } + + pub(crate) fn add_error(&mut self, e: &buck2_error::Error) { + self.error_tags.extend(e.tags()); + if let Some(category) = e.get_category() { + self.error_category = Some(category.combine(self.error_category.take())); + } + self.errors += 1; + } + + pub(crate) fn to_error(&self) -> Option { + if self.errors == 0 { + return None; + } + // Simpler error so that we don't print long errors twice (when exiting buck2) + let package_str = if self.errors == 1 { + "package" + } else { + "packages" + }; + + #[derive(buck2_error::Error, Debug)] + enum TargetsError { + #[error("Failed to parse {0} {1}")] + FailedToParse(u64, &'static str), + } + + let mut e = buck2_error::Error::from(TargetsError::FailedToParse(self.errors, package_str)); + e = e.tag(self.error_tags.iter().copied()); + if let Some(category) = self.error_category { + e = e.context(category); + } + Some(e.into()) + } } struct StatsFormat; diff --git a/app/buck2_server_commands/src/commands/targets/mod.rs b/app/buck2_server_commands/src/commands/targets/mod.rs index e3af1d2b90cf9..589fa099a386b 100644 --- a/app/buck2_server_commands/src/commands/targets/mod.rs +++ b/app/buck2_server_commands/src/commands/targets/mod.rs @@ -11,7 +11,6 @@ mod default; pub(crate) mod fmt; mod resolve_alias; mod streaming; - use std::fs::File; use std::io::BufWriter; use std::io::Write; @@ -27,8 +26,8 @@ use buck2_common::dice::cells::HasCellResolver; use buck2_core::pattern::pattern_type::TargetPatternExtra; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceTransaction; @@ -160,11 +159,6 @@ async fn targets( mut dice: DiceTransaction, request: &TargetsRequest, ) -> anyhow::Result { - // TODO(nmj): Rather than returning fully formatted data in the TargetsResponse, we should - // instead return structured data, and return *that* to the CLI. The CLI should - // then handle printing. The current approach is just a temporary hack to fix some - // issues with printing to stdout. - let cwd = server_ctx.working_dir(); let cell_resolver = dice.get_cell_resolver().await?; let parsed_target_patterns = parse_patterns_from_cli_args::( @@ -210,15 +204,16 @@ async fn targets( } else { let formatter = create_formatter(request, other)?; let client_ctx = request.client_context()?; - let target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut dice).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut dice) + .await?; let fs = server_ctx.project_root(); targets_batch( server_ctx, dice, &*formatter, parsed_target_patterns, - target_platform, + &global_cfg_options, TargetHashOptions::new(other, &cell_resolver, fs)?, other.keep_going, ) @@ -235,9 +230,3 @@ async fn targets( outputter.flush()?; Ok(response) } - -fn mk_error(errors: u64) -> anyhow::Error { - // Simpler error so that we don't print long errors twice (when exiting buck2) - let package_str = if errors == 1 { "package" } else { "packages" }; - anyhow::anyhow!("Failed to parse {} {}", errors, package_str) -} diff --git a/app/buck2_server_commands/src/commands/targets/streaming.rs b/app/buck2_server_commands/src/commands/targets/streaming.rs index c0f9e522640b2..69e68ef5b2b9c 100644 --- a/app/buck2_server_commands/src/commands/targets/streaming.rs +++ b/app/buck2_server_commands/src/commands/targets/streaming.rs @@ -36,7 +36,6 @@ use buck2_server_ctx::ctx::ServerCommandContextTrait; use dice::DiceComputations; use dice::DiceTransaction; use dupe::Dupe; -use dupe::IterDupedExt; use futures::future::FutureExt; use futures::Stream; use futures::StreamExt; @@ -49,7 +48,6 @@ use tokio::sync::Semaphore; use crate::commands::targets::fmt::Stats; use crate::commands::targets::fmt::TargetFormatter; use crate::commands::targets::fmt::TargetInfo; -use crate::commands::targets::mk_error; use crate::commands::targets::Outputter; use crate::target_hash::TargetHashes; @@ -100,7 +98,7 @@ pub(crate) async fn targets_streaming( load_targets(&ctx, package.dupe(), spec, cached, keep_going).await }; let mut show_err = |err| { - res.stats.errors += 1; + res.stats.add_error(err); let mut stderr = String::new(); formatter.package_error( package.dupe(), @@ -113,7 +111,7 @@ pub(crate) async fn targets_streaming( match targets { Ok((eval_result, targets, err)) => { if let Some(err) = err { - show_err(&err); + show_err(&err.into()); formatter.separator(&mut res.stdout); } res.stats.success += 1; @@ -137,7 +135,7 @@ pub(crate) async fn targets_streaming( } formatter.target( TargetInfo { - node, + node: node.as_ref(), target_hash: fast_hash.map(|fast| { TargetHashes::compute_immediate_one(node, fast) }), @@ -148,7 +146,7 @@ pub(crate) async fn targets_streaming( } } Err(err) => { - show_err(&err); + show_err(&err.into()); } } anyhow::Ok(res) @@ -175,7 +173,9 @@ pub(crate) async fn targets_streaming( if let Some(stderr) = &res.stderr { server_ctx.stderr()?.write_all(stderr.as_bytes())?; if !keep_going { - return Err(mk_error(stats.errors)); + return Err(stats + .to_error() + .expect("Result only has a stderr if there were errors")); } } if !res.stdout.is_empty() { @@ -303,7 +303,7 @@ async fn load_targets( .partition_map(|(target, TargetPatternExtra)| { match result.targets().get(target.as_ref()) { None => Either::Left(target), - Some(x) => Either::Right(x.dupe()), + Some(x) => Either::Right(x.to_owned()), } }); let err = if miss.is_empty() { @@ -314,13 +314,13 @@ async fn load_targets( Ok((result, targets, err)) } else { let targets = targets.into_try_map(|(target, TargetPatternExtra)| { - anyhow::Ok(result.resolve_target(target.as_ref())?.dupe()) + anyhow::Ok(result.resolve_target(target.as_ref())?.to_owned()) })?; Ok((result, targets, None)) } } PackageSpec::All => { - let targets = result.targets().values().duped().collect(); + let targets = result.targets().values().map(|t| t.to_owned()).collect(); Ok((result, targets, None)) } } diff --git a/app/buck2_server_commands/src/commands/targets_show_outputs.rs b/app/buck2_server_commands/src/commands/targets_show_outputs.rs index 18a2bb2582511..686886e27a240 100644 --- a/app/buck2_server_commands/src/commands/targets_show_outputs.rs +++ b/app/buck2_server_commands/src/commands/targets_show_outputs.rs @@ -19,6 +19,7 @@ use buck2_cli_proto::TargetsRequest; use buck2_cli_proto::TargetsShowOutputsResponse; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::file_ops::HasFileOps; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::pattern::resolve::resolve_target_patterns; use buck2_common::pattern::resolve::ResolvedPattern; use buck2_core::cells::CellResolver; @@ -36,8 +37,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use dice::DiceComputations; @@ -102,8 +103,8 @@ async fn targets_show_outputs( let cell_resolver = ctx.get_cell_resolver().await?; let client_ctx = request.client_context()?; - let target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; let parsed_patterns = parse_patterns_from_cli_args::( &mut ctx, @@ -118,7 +119,7 @@ async fn targets_show_outputs( for targets_artifacts in retrieve_targets_artifacts_from_patterns( &ctx, - &target_platform, + &global_cfg_options, &parsed_patterns, &cell_resolver, ) @@ -140,37 +141,30 @@ async fn targets_show_outputs( async fn retrieve_targets_artifacts_from_patterns( ctx: &DiceComputations, - global_target_platform: &Option, + global_cfg_options: &GlobalCfgOptions, parsed_patterns: &[ParsedPattern], cell_resolver: &CellResolver, ) -> anyhow::Result> { let resolved_pattern = resolve_target_patterns(cell_resolver, parsed_patterns, &ctx.file_ops()).await?; - retrieve_artifacts_for_targets(ctx, resolved_pattern, global_target_platform.to_owned()).await + retrieve_artifacts_for_targets(ctx, resolved_pattern, global_cfg_options).await } async fn retrieve_artifacts_for_targets( ctx: &DiceComputations, spec: ResolvedPattern, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result> { let futs: FuturesUnordered<_> = spec .specs .into_iter() .map(|(package, spec)| { - let global_target_platform = global_target_platform.dupe(); async move { { let res = ctx.get_interpreter_results(package.dupe()).await?; - retrieve_artifacts_for_spec( - ctx, - package.dupe(), - spec, - global_target_platform, - res, - ) - .await + retrieve_artifacts_for_spec(ctx, package.dupe(), spec, global_cfg_options, res) + .await } } .boxed() @@ -191,18 +185,18 @@ async fn retrieve_artifacts_for_spec( ctx: &DiceComputations, package: PackageLabel, spec: PackageSpec, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, res: Arc, ) -> anyhow::Result> { let available_targets = res.targets(); - let todo_targets: Vec<(ProvidersLabel, Option)> = match spec { + let todo_targets: Vec<(ProvidersLabel, &GlobalCfgOptions)> = match spec { PackageSpec::All => available_targets .keys() .map(|t| { ( ProvidersLabel::default_for(TargetLabel::new(package.dupe(), t)), - global_target_platform.dupe(), + global_cfg_options, ) }) .collect(), @@ -213,7 +207,7 @@ async fn retrieve_artifacts_for_spec( targets.into_map(|(target_name, providers)| { ( providers.into_providers_label(package.dupe(), target_name.as_ref()), - global_target_platform.dupe(), + global_cfg_options, ) }) } @@ -221,8 +215,8 @@ async fn retrieve_artifacts_for_spec( let mut futs: FuturesUnordered<_> = todo_targets .into_iter() - .map(|(providers_label, target_platform)| { - retrieve_artifacts_for_provider_label(ctx, providers_label, target_platform) + .map(|(providers_label, cfg_flags)| { + retrieve_artifacts_for_provider_label(ctx, providers_label, cfg_flags) }) .collect(); @@ -237,10 +231,10 @@ async fn retrieve_artifacts_for_spec( async fn retrieve_artifacts_for_provider_label( ctx: &DiceComputations, providers_label: ProvidersLabel, - target_platform: Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result { let providers_label = ctx - .get_configured_provider_label(&providers_label, target_platform.as_ref()) + .get_configured_provider_label(&providers_label, global_cfg_options) .await?; let providers = ctx diff --git a/app/buck2_server_commands/src/dot/mod.rs b/app/buck2_server_commands/src/dot/mod.rs index c525008004c45..614eb0fb3b9f5 100644 --- a/app/buck2_server_commands/src/dot/mod.rs +++ b/app/buck2_server_commands/src/dot/mod.rs @@ -11,7 +11,6 @@ //! //! Has a lot less features than or , //! but it's easier for us to match buck1's output with this simple implementation. -//! // TODO(cjhopman): while the `dot` crate is probably too opinionated, `tabbycat` looks nice and is // lower level so gives a lot of control (including control over ordering of node/edge statements). // It looks like we could use that, but it mostly would just handle the actual writing of the diff --git a/app/buck2_server_commands/src/dot/targets.rs b/app/buck2_server_commands/src/dot/targets.rs index 3767e15e3b46f..220e78ee56b31 100644 --- a/app/buck2_server_commands/src/dot/targets.rs +++ b/app/buck2_server_commands/src/dot/targets.rs @@ -13,6 +13,7 @@ use buck2_query::query::syntax::simple::eval::set::TargetSet; use regex::RegexSet; use starlark_map::small_map::SmallMap; +use crate::commands::query::query_target_ext::QueryCommandTarget; use crate::dot::DotDigraph; use crate::dot::DotEdge; use crate::dot::DotNode; @@ -26,7 +27,7 @@ pub struct DotTargetGraph { pub attributes: Option, } -impl<'a, T: QueryTarget> DotDigraph<'a> for DotTargetGraph { +impl<'a, T: QueryCommandTarget> DotDigraph<'a> for DotTargetGraph { type Node = DotTargetGraphNode<'a, T>; fn name(&self) -> &str { @@ -52,7 +53,7 @@ impl<'a, T: QueryTarget> DotDigraph<'a> for DotTargetGraph { // Only include edges to other nodes within the subgraph. if self.targets.contains(dep) { f(&DotEdge { - from: &node.0.node_ref().to_string(), + from: &node.0.node_key().to_string(), to: &dep.to_string(), })?; } @@ -61,7 +62,7 @@ impl<'a, T: QueryTarget> DotDigraph<'a> for DotTargetGraph { } } -impl<'a, T: QueryTarget> DotNode for DotTargetGraphNode<'a, T> { +impl<'a, T: QueryCommandTarget> DotNode for DotTargetGraphNode<'a, T> { fn attrs(&self) -> anyhow::Result { let extra = match &self.1.attributes { Some(attr_regex) => { @@ -91,6 +92,6 @@ impl<'a, T: QueryTarget> DotNode for DotTargetGraphNode<'a, T> { } fn id(&self) -> String { - self.0.node_ref().to_string() + self.0.node_key().to_string() } } diff --git a/app/buck2_server_commands/src/lib.rs b/app/buck2_server_commands/src/lib.rs index 16ea91de58aad..b9f3c27450fc0 100644 --- a/app/buck2_server_commands/src/lib.rs +++ b/app/buck2_server_commands/src/lib.rs @@ -13,6 +13,7 @@ #![feature(async_closure)] #![feature(box_patterns)] +#![feature(let_chains)] #![feature(try_blocks)] pub mod commands; diff --git a/app/buck2_server_commands/src/target_hash.rs b/app/buck2_server_commands/src/target_hash.rs index ba5da950cca8e..799a5b267786d 100644 --- a/app/buck2_server_commands/src/target_hash.rs +++ b/app/buck2_server_commands/src/target_hash.rs @@ -20,21 +20,21 @@ use buck2_common::dice::file_ops::HasFileOps; use buck2_common::file_ops::FileOps; use buck2_common::file_ops::PathMetadata; use buck2_common::file_ops::PathMetadataOrRedirection; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_core::cells::cell_path::CellPath; use buck2_core::cells::cell_path::CellPathRef; use buck2_core::package::PackageLabel; +use buck2_core::target::configured_or_unconfigured::ConfiguredOrUnconfiguredTargetLabel; use buck2_core::target::label::TargetLabel; use buck2_futures::spawn::spawn_cancellable; use buck2_futures::spawn::DropCancelFuture; use buck2_node::nodes::configured::ConfiguredTargetNode; use buck2_node::nodes::unconfigured::TargetNode; -use buck2_query::query::environment::ConfiguredOrUnconfiguredTargetLabel; use buck2_query::query::environment::QueryTarget; +use buck2_query::query::environment::QueryTargetDepsSuccessors; use buck2_query::query::syntax::simple::eval::set::TargetSet; use buck2_query::query::traversal::async_depth_first_postorder_traversal; use buck2_query::query::traversal::AsyncNodeLookup; -use buck2_query::query::traversal::AsyncTraversalDelegate; -use buck2_query::query::traversal::ChildVisitor; use dice::DiceComputations; use dice::DiceTransaction; use dupe::Dupe; @@ -193,7 +193,7 @@ pub trait TargetHashingTargetNode: QueryTarget { async fn get_target_nodes( dice: &DiceComputations, loaded_targets: Vec<(PackageLabel, anyhow::Result>)>, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result>; } @@ -206,9 +206,9 @@ impl TargetHashingTargetNode for ConfiguredTargetNode { async fn get_target_nodes( dice: &DiceComputations, loaded_targets: Vec<(PackageLabel, anyhow::Result>)>, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result> { - get_compatible_targets(dice, loaded_targets.into_iter(), global_target_platform).await + get_compatible_targets(dice, loaded_targets.into_iter(), global_cfg_options).await } } @@ -221,7 +221,7 @@ impl TargetHashingTargetNode for TargetNode { async fn get_target_nodes( _dice: &DiceComputations, loaded_targets: Vec<(PackageLabel, anyhow::Result>)>, - _global_target_platform: Option, + _global_cfg_options: &GlobalCfgOptions, ) -> anyhow::Result> { let mut target_set = TargetSet::new(); for (_package, result) in loaded_targets { @@ -256,102 +256,81 @@ impl TargetHashes { use_fast_hash: bool, ) -> anyhow::Result where - T::NodeRef: ConfiguredOrUnconfiguredTargetLabel, + T::Key: ConfiguredOrUnconfiguredTargetLabel, { - struct Delegate { - hashes: - HashMap>>>, - file_hasher: Option>, - use_fast_hash: bool, - dice: DiceTransaction, - } - - #[async_trait] - impl AsyncTraversalDelegate for Delegate { - fn visit(&mut self, target: T) -> anyhow::Result<()> { - // this is postorder, so guaranteed that all deps have futures already. - let dep_futures: Vec<_> = target - .deps() - .map(|dep| { - self.hashes.get(dep).cloned().ok_or_else(|| { - TargetHashError::DependencyCycle( - dep.clone().to_string(), - target.node_ref().to_string(), - ) - }) + let mut hashes: HashMap< + T::Key, + Shared>>, + > = HashMap::new(); + + let visit = |target: T| { + // this is postorder, so guaranteed that all deps have futures already. + let dep_futures: Vec<_> = target + .deps() + .map(|dep| { + hashes.get(dep).cloned().ok_or_else(|| { + TargetHashError::DependencyCycle( + dep.clone().to_string(), + target.node_key().to_string(), + ) }) - .collect::, TargetHashError>>()?; - - let file_hasher = self.file_hasher.dupe(); - let dice = self.dice.dupe(); - - let use_fast_hash = self.use_fast_hash; - // we spawn off the hash computation since it can't be done in visit directly. Even if it could, - // this allows us to start the computations for dependents before finishing the computation for a node. - self.hashes.insert( - target.node_ref().clone(), - spawn_cancellable( - |_| { - async move { - let mut hasher = TargetHashes::new_hasher(use_fast_hash); - TargetHashes::hash_node(&target, &mut *hasher); - - let mut input_futs = Vec::new(); - if let Some(file_hasher) = file_hasher { - target.inputs_for_each(|cell_path| { - let file_hasher = file_hasher.dupe(); - input_futs.push(async move { - let file_hash = file_hasher.hash_path(&cell_path).await; - (cell_path, file_hash) - }); - anyhow::Ok(()) - })?; - } - - let (dep_hashes, input_hashes) = - join!(join_all(dep_futures), join_all(input_futs)); - - TargetHashes::hash_deps(dep_hashes, &mut *hasher)?; - TargetHashes::hash_files(input_hashes, &mut *hasher)?; - - Ok(hasher.finish_u128()) + }) + .collect::, TargetHashError>>()?; + + let file_hasher = file_hasher.dupe(); + let dice = dice.dupe(); + + // we spawn off the hash computation since it can't be done in visit directly. Even if it could, + // this allows us to start the computations for dependents before finishing the computation for a node. + hashes.insert( + target.node_key().clone(), + spawn_cancellable( + |_| { + async move { + let mut hasher = TargetHashes::new_hasher(use_fast_hash); + TargetHashes::hash_node(&target, &mut *hasher); + + let mut input_futs = Vec::new(); + if let Some(file_hasher) = file_hasher { + target.inputs_for_each(|cell_path| { + let file_hasher = file_hasher.dupe(); + input_futs.push(async move { + let file_hash = file_hasher.hash_path(&cell_path).await; + (cell_path, file_hash) + }); + anyhow::Ok(()) + })?; } - .boxed() - }, - &*dice.per_transaction_data().spawner, - dice.per_transaction_data(), - ) - .into_drop_cancel() - .shared(), - ); - Ok(()) - } + let (dep_hashes, input_hashes) = + join!(join_all(dep_futures), join_all(input_futs)); - async fn for_each_child( - &mut self, - target: &T, - func: &mut dyn ChildVisitor, - ) -> anyhow::Result<()> { - for dep in target.deps() { - func.visit(dep.clone())?; - } + TargetHashes::hash_deps(dep_hashes, &mut *hasher)?; + TargetHashes::hash_files(input_hashes, &mut *hasher)?; - Ok(()) - } - } + Ok(hasher.finish_u128()) + } + .boxed() + }, + &*dice.per_transaction_data().spawner, + dice.per_transaction_data(), + ) + .into_drop_cancel() + .shared(), + ); - let mut delegate = Delegate:: { - hashes: HashMap::new(), - file_hasher, - use_fast_hash, - dice, + Ok(()) }; - async_depth_first_postorder_traversal(&lookup, targets.iter_names(), &mut delegate).await?; + async_depth_first_postorder_traversal( + &lookup, + targets.iter_names(), + QueryTargetDepsSuccessors, + visit, + ) + .await?; - let mut futures: FuturesUnordered<_> = delegate - .hashes + let mut futures: FuturesUnordered<_> = hashes .into_iter() .map(|(target, fut)| async move { (target, fut.await) }) .collect(); @@ -383,7 +362,7 @@ impl TargetHashes { use_fast_hash: bool, ) -> anyhow::Result where - T::NodeRef: ConfiguredOrUnconfiguredTargetLabel, + T::Key: ConfiguredOrUnconfiguredTargetLabel, { let hashing_futures: Vec<_> = targets .into_iter() @@ -412,7 +391,7 @@ impl TargetHashes { hasher.finish_u128() }; ( - target.node_ref().unconfigured_label().dupe(), + target.node_key().unconfigured_label().dupe(), hash_result.map_err(buck2_error::Error::from), ) } @@ -436,15 +415,15 @@ impl TargetHashes { dice: DiceTransaction, lookup: L, targets: Vec<(PackageLabel, anyhow::Result>)>, - global_target_platform: Option, + global_cfg_options: &GlobalCfgOptions, file_hash_mode: TargetHashesFileMode, use_fast_hash: bool, target_hash_recursive: bool, ) -> anyhow::Result where - T::NodeRef: ConfiguredOrUnconfiguredTargetLabel, + T::Key: ConfiguredOrUnconfiguredTargetLabel, { - let targets = T::get_target_nodes(&dice, targets, global_target_platform).await?; + let targets = T::get_target_nodes(&dice, targets, global_cfg_options).await?; let file_hasher = Self::new_file_hasher(dice.dupe(), file_hash_mode); if target_hash_recursive { Self::compute_recursive_target_hashes(dice, lookup, targets, file_hasher, use_fast_hash) diff --git a/app/buck2_server_ctx/src/pattern.rs b/app/buck2_server_ctx/src/pattern.rs index 3a115f6aaa33d..76c879bfd5d6d 100644 --- a/app/buck2_server_ctx/src/pattern.rs +++ b/app/buck2_server_ctx/src/pattern.rs @@ -9,6 +9,7 @@ use buck2_cli_proto::ClientContext; use buck2_common::dice::cells::HasCellResolver; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::target_aliases::BuckConfigTargetAliasResolver; use buck2_common::target_aliases::HasTargetAliasResolver; use buck2_core::cells::cell_path::CellPath; @@ -16,7 +17,6 @@ use buck2_core::cells::CellResolver; use buck2_core::fs::project_rel_path::ProjectRelativePath; use buck2_core::pattern::pattern_type::PatternType; use buck2_core::pattern::ParsedPattern; -use buck2_core::target::label::TargetLabel; use dice::DiceComputations; use gazebo::prelude::*; @@ -72,34 +72,27 @@ pub async fn parse_patterns_from_cli_args( target_patterns.try_map(|value| parser.parse_pattern(&value.value)) } -/// Extract target configuration (platform) label from [`ClientContext`]. -pub async fn target_platform_from_client_context( - client_ctx: &ClientContext, +/// Extract target configuration components from [`ClientContext`]. +pub async fn global_cfg_options_from_client_context( + client_context: &ClientContext, server_ctx: &dyn ServerCommandContextTrait, dice_ctx: &mut DiceComputations, -) -> anyhow::Result> { - target_platform_from_client_context_impl( - client_ctx, - &dice_ctx.get_cell_resolver().await?, - server_ctx.working_dir(), - ) - .await -} - -async fn target_platform_from_client_context_impl( - client_context: &ClientContext, - cell_resolver: &CellResolver, - working_dir: &ProjectRelativePath, -) -> anyhow::Result> { +) -> anyhow::Result { + let cell_resolver: &CellResolver = &dice_ctx.get_cell_resolver().await?; + let working_dir: &ProjectRelativePath = server_ctx.working_dir(); let cwd = cell_resolver.get_cell_path(working_dir)?; - let target_platform = &client_context.target_platform; - if !target_platform.is_empty() { - Ok(Some( + let target_platform_label = if !target_platform.is_empty() { + Some( ParsedPattern::parse_precise(target_platform, cwd.cell(), cell_resolver)? .as_target_label(target_platform)?, - )) + ) } else { - Ok(None) - } + None + }; + + Ok(GlobalCfgOptions { + target_platform: target_platform_label, + cli_modifiers: client_context.cli_modifiers.clone().into(), + }) } diff --git a/app/buck2_starlark/src/debug.rs b/app/buck2_starlark/src/debug.rs index 8f8083842653f..dcb10a74d72e7 100644 --- a/app/buck2_starlark/src/debug.rs +++ b/app/buck2_starlark/src/debug.rs @@ -201,7 +201,7 @@ impl StreamingCommand for StarlarkDebugAttachCommand { Ok(()) } - async fn handle_error(&mut self, error: &anyhow::Error) -> anyhow::Result<()> { + async fn handle_error(&mut self, error: &buck2_error::Error) -> anyhow::Result<()> { self.write_console(&format!( "buck2 starlark-attach debugserver error: {}", error diff --git a/app/buck2_test/BUCK b/app/buck2_test/BUCK index 118ca339818da..9dcd48fe0a399 100644 --- a/app/buck2_test/BUCK +++ b/app/buck2_test/BUCK @@ -44,6 +44,7 @@ rust_library( "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_downward_api:buck2_downward_api", "//buck2/app/buck2_error:buck2_error", + "//buck2/app/buck2_error_derive:buck2_error_derive", "//buck2/app/buck2_events:buck2_events", "//buck2/app/buck2_execute:buck2_execute", "//buck2/app/buck2_execute_impl:buck2_execute_impl", diff --git a/app/buck2_test/Cargo.toml b/app/buck2_test/Cargo.toml index 9b98f724cf1ec..28795b8ff0212 100644 --- a/app/buck2_test/Cargo.toml +++ b/app/buck2_test/Cargo.toml @@ -35,6 +35,7 @@ buck2_core = { workspace = true } buck2_data = { workspace = true } buck2_downward_api = { workspace = true } buck2_error = { workspace = true } +buck2_error_derive = { workspace = true } buck2_events = { workspace = true } buck2_execute = { workspace = true } buck2_execute_impl = { workspace = true } diff --git a/app/buck2_test/src/command.rs b/app/buck2_test/src/command.rs index 6daf064c8858f..4ebf3807d6f4c 100644 --- a/app/buck2_test/src/command.rs +++ b/app/buck2_test/src/command.rs @@ -12,6 +12,7 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use anyhow::Context; use async_trait::async_trait; @@ -27,9 +28,12 @@ use buck2_cli_proto::TestResponse; use buck2_common::dice::cells::HasCellResolver; use buck2_common::dice::file_ops::HasFileOps; use buck2_common::events::HasEvents; +use buck2_common::global_cfg_options::GlobalCfgOptions; use buck2_common::legacy_configs::dice::HasLegacyConfigs; use buck2_common::liveliness_observer::LivelinessGuard; use buck2_common::liveliness_observer::LivelinessObserver; +use buck2_common::liveliness_observer::LivelinessObserverExt; +use buck2_common::liveliness_observer::TimeoutLivelinessObserver; use buck2_common::pattern::resolve::resolve_target_patterns; use buck2_common::pattern::resolve::ResolvedPattern; use buck2_core::buck2_env; @@ -60,8 +64,8 @@ use buck2_node::target_calculation::ConfiguredTargetCalculation; use buck2_server_ctx::ctx::ServerCommandContextTrait; use buck2_server_ctx::partial_result_dispatcher::NoPartialResult; use buck2_server_ctx::partial_result_dispatcher::PartialResultDispatcher; +use buck2_server_ctx::pattern::global_cfg_options_from_client_context; use buck2_server_ctx::pattern::parse_patterns_from_cli_args; -use buck2_server_ctx::pattern::target_platform_from_client_context; use buck2_server_ctx::template::run_server_command; use buck2_server_ctx::template::ServerCommandTemplate; use buck2_server_ctx::test_command::TEST_COMMAND; @@ -110,11 +114,6 @@ struct TestOutcome { impl TestOutcome { pub(crate) fn exit_code(&self) -> anyhow::Result> { - if !self.errors.is_empty() { - // Some tests failed to build. Send `None` back to - // the client to delegate the exit code generation. - return Ok(None); - } self.executor_report .exit_code .context("Test executor did not provide an exit code") @@ -204,6 +203,11 @@ impl TestStatuses { } } +#[derive(Debug, buck2_error_derive::Error)] +#[buck2(user, typ = UserDeadlineExpired)] +#[error("This test run exceeded the deadline that was provided")] +struct DeadlineExpired; + async fn test_command( ctx: &dyn ServerCommandContextTrait, partial_result_dispatcher: PartialResultDispatcher, @@ -268,8 +272,8 @@ async fn test( let working_dir_cell = cell_resolver.find(cwd)?; let client_ctx = request.client_context()?; - let global_target_platform = - target_platform_from_client_context(client_ctx, server_ctx, &mut ctx).await?; + let global_cfg_options = + global_cfg_options_from_client_context(client_ctx, server_ctx, &mut ctx).await?; // Get the test runner from the config. Note that we use a different key from v1 since the API // is completely different, so there is not expectation that the same binary works for both. @@ -282,7 +286,8 @@ async fn test( Some(config) => { let test_executor = post_process_test_executor(config.as_ref()) .with_context(|| format!("Invalid `test.v2_test_executor`: {}", config))?; - let test_executor_args = Vec::new(); + let test_executor_args = + vec!["--buck-trace-id".to_owned(), client_ctx.trace_id.clone()]; (test_executor, test_executor_args) } None => { @@ -321,10 +326,18 @@ async fn test( .build_opts .as_ref() .expect("should have build options"); + + let timeout = request + .timeout + .as_ref() + .map(|t| t.clone().try_into()) + .transpose() + .context("Invalid `duration`")?; + let test_outcome = test_targets( ctx, resolved_pattern, - global_target_platform, + global_cfg_options, request.test_executor_args.clone(), Arc::new(TestLabelFiltering::new( request.included_labels.clone(), @@ -338,6 +351,7 @@ async fn test( working_dir_cell, build_opts.skip_incompatible_targets, MissingTargetBehavior::from_skip(build_opts.skip_missing_targets), + timeout, ) .await?; @@ -402,7 +416,7 @@ async fn test( async fn test_targets( ctx: DiceTransaction, pattern: ResolvedPattern, - global_target_platform: Option, + global_cfg_options: GlobalCfgOptions, external_runner_args: Vec, label_filtering: Arc, launcher: &dyn ExecutorLauncher, @@ -411,9 +425,17 @@ async fn test_targets( working_dir_cell: CellName, skip_incompatible_targets: bool, missing_target_behavior: MissingTargetBehavior, + timeout: Option, ) -> anyhow::Result { let session = Arc::new(session); - let (liveliness_observer, _guard) = LivelinessGuard::create(); + + let (mut liveliness_observer, _guard) = LivelinessGuard::create(); + let timeout_observer = timeout.map(|timeout| { + Arc::new(TimeoutLivelinessObserver::new(timeout)) as Arc + }); + if let Some(timeout_observer) = &timeout_observer { + liveliness_observer = Arc::new(liveliness_observer.and(timeout_observer.dupe())) as _; + } let tpx_args = { let mut args = vec![ @@ -451,6 +473,7 @@ async fn test_targets( let test_server = tokio::spawn({ let test_status_sender = test_status_sender.clone(); + let liveliness_observer = liveliness_observer.dupe(); with_dispatcher_async( ctx.per_transaction_data().get_dispatcher().dupe(), // NOTE: This is will cancel if the liveliness guard indicates we should. @@ -475,7 +498,7 @@ async fn test_targets( let mut driver = TestDriver::new(TestDriverState { ctx: &ctx, label_filtering: &label_filtering, - global_target_platform: &global_target_platform, + global_cfg_options: &global_cfg_options, session: &session, test_executor: &test_executor, cell_resolver: &cell_resolver, @@ -561,11 +584,17 @@ async fn test_targets( .await .context("Failed to collect executor report")??; - let errors = build_errors + let mut errors = build_errors .iter() .map(create_error_report) .unique_by(|e| e.message.clone()) - .collect(); + .collect::>(); + + if let Some(timeout_observer) = timeout_observer { + if !timeout_observer.is_alive().await { + errors.push(create_error_report(&DeadlineExpired.into())); + } + } Ok(TestOutcome { errors, @@ -594,7 +623,7 @@ enum TestDriverTask { pub(crate) struct TestDriverState<'a, 'e> { ctx: &'a DiceComputations, label_filtering: &'a Arc, - global_target_platform: &'a Option, + global_cfg_options: &'a GlobalCfgOptions, session: &'a TestSession, test_executor: &'a Arc, cell_resolver: &'a CellResolver, @@ -713,7 +742,7 @@ impl<'a, 'e> TestDriver<'a, 'e> { let fut = async move { let label = state .ctx - .get_configured_provider_label(&label, state.global_target_platform.as_ref()) + .get_configured_provider_label(&label, state.global_cfg_options) .await?; let node = state.ctx.get_configured_target_node(label.target()).await?; diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index eba0d1245c383..3c480590d2670 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -287,7 +287,7 @@ impl<'a> BuckTestOrchestrator<'a> { .await?; let (stdout, stderr, status, timing, execution_kind, outputs) = self - .execute_shared(&test_target, metadata, &test_executor, execution_request) + .execute_request(&test_target, metadata, &test_executor, execution_request) .await?; self.require_alive().await?; @@ -525,7 +525,8 @@ impl<'b> BuckTestOrchestrator<'b> { Ok(executor_preference) } - async fn execute_shared( + /// Core request execution logic. + async fn execute_request( &self, test_target: &ConfiguredProvidersLabel, metadata: DisplayMetadata, diff --git a/app/buck2_test_api/src/data/convert.rs b/app/buck2_test_api/src/data/convert.rs index 862647d4e8f0e..f08974da7eeef 100644 --- a/app/buck2_test_api/src/data/convert.rs +++ b/app/buck2_test_api/src/data/convert.rs @@ -608,7 +608,7 @@ impl TryInto for ExecutionResult2 { .into_iter() .map(|(k, v)| { Ok(buck2_test_proto::OutputEntry { - declared_output: Some(k.try_into().context("Invalid `declared_output`")?), + declared_output: Some(k.into()), output: Some(v.try_into().context("Invalid `output`")?), }) }) @@ -769,10 +769,7 @@ impl TryInto for TestExecutable { }) .collect::>()?; - let pre_create_dirs = self - .pre_create_dirs - .into_try_map(|i| i.try_into()) - .context("Invalid `pre_create_dirs`")?; + let pre_create_dirs = self.pre_create_dirs.into_map(|i| i.into()); Ok(buck2_test_proto::TestExecutable { ui_prints, diff --git a/app/buck2_transition/src/transition/calculation_apply_transition.rs b/app/buck2_transition/src/transition/calculation_apply_transition.rs index bc4e7d89e42c8..062b03b84fe42 100644 --- a/app/buck2_transition/src/transition/calculation_apply_transition.rs +++ b/app/buck2_transition/src/transition/calculation_apply_transition.rs @@ -31,7 +31,7 @@ use buck2_interpreter::starlark_profiler::StarlarkProfilerOrInstrumentation; use buck2_node::attrs::coerced_attr::CoercedAttr; use buck2_node::attrs::display::AttrDisplayWithContextExt; use buck2_node::attrs::inspect_options::AttrInspectOptions; -use buck2_node::nodes::unconfigured::TargetNode; +use buck2_node::nodes::unconfigured::TargetNodeRef; use derive_more::Display; use dice::DiceComputations; use dice::Key; @@ -235,7 +235,7 @@ impl TransitionCalculation for TransitionCalculationImpl { async fn apply_transition( &self, ctx: &DiceComputations, - target_node: &TargetNode, + target_node: TargetNodeRef<'_>, cfg: &ConfigurationData, transition_id: &TransitionId, ) -> anyhow::Result> { diff --git a/app/buck2_util/BUCK b/app/buck2_util/BUCK index d985a40f323f6..84e5ebee06369 100644 --- a/app/buck2_util/BUCK +++ b/app/buck2_util/BUCK @@ -36,6 +36,7 @@ rust_library( "fbsource//third-party/rust:futures", "fbsource//third-party/rust:serde", "fbsource//third-party/rust:static_assertions", + "fbsource//third-party/rust:sysinfo", "fbsource//third-party/rust:tokio", "fbsource//third-party/rust:tracing", "fbsource//third-party/rust:triomphe", diff --git a/app/buck2_util/Cargo.toml b/app/buck2_util/Cargo.toml index 8fdc5acba8444..44d286d4c37ce 100644 --- a/app/buck2_util/Cargo.toml +++ b/app/buck2_util/Cargo.toml @@ -15,6 +15,7 @@ anyhow = { workspace = true } dupe = { workspace = true } futures = { workspace = true } starlark_map = { workspace = true } +sysinfo = { workspace = true } tracing = { workspace = true } triomphe = { workspace = true } diff --git a/app/buck2_util/src/lib.rs b/app/buck2_util/src/lib.rs index 9e3e432b811fe..6c80852662a64 100644 --- a/app/buck2_util/src/lib.rs +++ b/app/buck2_util/src/lib.rs @@ -20,6 +20,9 @@ pub mod late_binding; pub mod process; pub mod process_stats; pub mod rtabort; +pub mod self_ref; pub mod system_stats; pub mod thin_box; +pub mod threads; +pub mod tokio_runtime; pub mod truncate; diff --git a/app/buck2_util/src/self_ref.rs b/app/buck2_util/src/self_ref.rs new file mode 100644 index 0000000000000..633bb4187bf7a --- /dev/null +++ b/app/buck2_util/src/self_ref.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::convert::Infallible; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use allocative::Allocative; + +/// Describing data that can be stored in `SelfRef`. +pub trait RefData: 'static { + type Data<'a>: 'a; +} + +/// Self-referential struct. +#[derive(Allocative)] +#[allocative(bound = "D: RefData")] +pub struct SelfRef { + #[allocative(skip)] // TODO(nga): do not skip. + data: D::Data<'static>, + // Owner must be placed after `data` to ensure that `data` is dropped before `owner`. + // Owner must be in `Arc` (or `Rc`) because + // - pointers stay valid when `SelfRef` is moved. + // - it cannot be `Box` because it would violate aliasing rules + owner: Arc, +} + +impl Debug for SelfRef +where + D: RefData, + for<'a> D::Data<'a>: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SelfRef") + .field("data", self.data()) + .finish_non_exhaustive() + } +} + +impl SelfRef { + pub fn try_new( + owner: O, + data: impl for<'a> FnOnce(&'a O) -> Result, E>, + ) -> Result { + let owner: Arc = Arc::new(owner); + let data = data(&owner)?; + let data = unsafe { std::mem::transmute::, D::Data<'static>>(data) }; + Ok(SelfRef { owner, data }) + } + + pub fn new( + owner: O, + data: impl for<'a> FnOnce(&'a O) -> D::Data<'a>, + ) -> Self { + match Self::try_new(owner, |f| Ok::<_, Infallible>(data(f))) { + Ok(x) => x, + Err(e) => match e {}, + } + } + + #[inline] + pub fn data(&self) -> &D::Data<'_> { + unsafe { std::mem::transmute::<&D::Data<'static>, &D::Data<'_>>(&self.data) } + } +} diff --git a/app/buck2_util/src/system_stats.rs b/app/buck2_util/src/system_stats.rs index 4dd04d3b6aea6..956bf44822010 100644 --- a/app/buck2_util/src/system_stats.rs +++ b/app/buck2_util/src/system_stats.rs @@ -33,3 +33,24 @@ impl UnixSystemStats { None } } + +pub fn system_memory_stats() -> u64 { + use sysinfo::RefreshKind; + use sysinfo::System; + use sysinfo::SystemExt; + + let system = System::new_with_specifics(RefreshKind::new().with_memory()); + system.total_memory() +} + +#[cfg(test)] +mod tests { + use crate::system_stats::system_memory_stats; + + #[test] + fn get_system_memory_stats() { + let total_mem = system_memory_stats(); + // sysinfo returns zero when fails to retrieve data + assert!(total_mem > 0); + } +} diff --git a/app/buck2_util/src/threads.rs b/app/buck2_util/src/threads.rs new file mode 100644 index 0000000000000..a2051229ed072 --- /dev/null +++ b/app/buck2_util/src/threads.rs @@ -0,0 +1,181 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::cell::Cell; +use std::future::Future; +use std::hint; +use std::pin::pin; +use std::pin::Pin; +use std::task::Poll; +use std::thread; + +use anyhow::Context; + +/// Default stack size for buck2. +/// +/// We want to be independent of possible future changes to the default stack size in Rust. +pub(crate) const THREAD_DEFAULT_STACK_SIZE: usize = 2 << 20; + +pub fn thread_spawn(name: &str, code: F) -> std::io::Result> +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ + thread::Builder::new() + .stack_size(THREAD_DEFAULT_STACK_SIZE) + .name(name.to_owned()) + .spawn(move || { + on_thread_start(); + let r = code(); + on_thread_stop(); + r + }) +} + +pub(crate) fn stack_pointer() -> *const () { + let mut x: u32 = 0; + hint::black_box(&mut x as *const u32 as *const ()) +} + +#[derive(Copy, Clone)] +struct ValidStackRange { + start: *const (), + end: *const (), +} + +impl ValidStackRange { + fn full_range() -> ValidStackRange { + let start = usize::MAX as *const (); + let end = usize::MIN as *const (); + ValidStackRange { start, end } + } +} + +thread_local! { + static STACK_RANGE: Cell> = Cell::new(None); +} + +pub(crate) fn on_thread_start() { + assert!( + STACK_RANGE.get().is_none(), + "stack range must not be set in a new thread" + ); + let stack_pointer = stack_pointer(); + // Stack grows downwards. So we add to the start and subtract from the end. + // Add a little bit to the start because we don't really know where the stack starts. + let start = (stack_pointer as usize).checked_add(0x1000).unwrap() as *const (); + // Subtract 3/4 to catch stack overflow before program crashes. + let end = (stack_pointer as usize) + .checked_sub(THREAD_DEFAULT_STACK_SIZE / 4 * 3) + .unwrap() as *const (); + let stack_range = ValidStackRange { start, end }; + STACK_RANGE.set(Some(stack_range)); +} + +pub(crate) fn on_thread_stop() { + let range = STACK_RANGE.replace(None); + assert!(range.is_some(), "stack range must be set in a thread"); +} + +pub fn check_stack_overflow() -> anyhow::Result<()> { + let stack_range = STACK_RANGE + .get() + .context("stack range not set (internal error)")?; + let stack_pointer = stack_pointer(); + if stack_pointer > stack_range.start { + return Err(anyhow::anyhow!( + "stack underflow, should not happen (internal error)" + )); + } + if stack_pointer < stack_range.end { + // TODO(nga): need to tag this error, but we don't have tags in `buck2_util`. + return Err(anyhow::anyhow!("stack overflow (internal error)")); + } + Ok(()) +} + +#[must_use] +pub struct IgnoreStackOverflowChecksForCurrentThread { + prev: Option, +} + +impl Drop for IgnoreStackOverflowChecksForCurrentThread { + fn drop(&mut self) { + STACK_RANGE.set(self.prev.take()); + } +} + +/// For tests. +pub fn ignore_stack_overflow_checks_for_current_thread() -> IgnoreStackOverflowChecksForCurrentThread +{ + let prev = STACK_RANGE.replace(Some(ValidStackRange::full_range())); + IgnoreStackOverflowChecksForCurrentThread { prev } +} + +/// For tests. +pub async fn ignore_stack_overflow_checks_for_future(f: F) -> F::Output { + let f = pin!(f); + + struct IgnoreStackOverflowChecksForFuture<'a, F> { + f: Pin<&'a mut F>, + } + + impl<'a, F: Future> Future for IgnoreStackOverflowChecksForFuture<'a, F> { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let _ignore = ignore_stack_overflow_checks_for_current_thread(); + self.f.as_mut().poll(cx) + } + } + + IgnoreStackOverflowChecksForFuture { f }.await +} + +#[cfg(test)] +pub(crate) mod tests { + use std::hint; + + use crate::threads::check_stack_overflow; + use crate::threads::thread_spawn; + + pub(crate) fn recursive_function(frames: u32) -> anyhow::Result<()> { + let Some(frames) = frames.checked_sub(1) else { + return Ok(()); + }; + + check_stack_overflow()?; + + // Allocate a string on the stack so the compiler won't optimize the recursion away. + let mut x = String::new(); + hint::black_box(&mut x); + recursive_function(frames)?; + hint::black_box(&mut x); + Ok(()) + } + + #[test] + fn test_catch_stack_overflow() { + let error = thread_spawn("test", || recursive_function(u32::MAX)) + .unwrap() + .join() + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("stack overflow"), "{error:?}"); + } + + #[test] + fn test_no_stack_overflow() { + let () = thread_spawn("test", || recursive_function(1000)) + .unwrap() + .join() + .unwrap() + .unwrap(); + } +} diff --git a/app/buck2_util/src/tokio_runtime.rs b/app/buck2_util/src/tokio_runtime.rs new file mode 100644 index 0000000000000..9256b15a79aff --- /dev/null +++ b/app/buck2_util/src/tokio_runtime.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use tokio::runtime::Builder; + +use crate::threads::on_thread_start; +use crate::threads::on_thread_stop; +use crate::threads::THREAD_DEFAULT_STACK_SIZE; + +pub fn new_tokio_runtime(thread_name: &str) -> Builder { + let mut builder = Builder::new_multi_thread(); + builder.thread_stack_size(THREAD_DEFAULT_STACK_SIZE); + builder.thread_name(thread_name); + builder.on_thread_start(on_thread_start); + builder.on_thread_stop(on_thread_stop); + builder +} + +#[cfg(test)] +mod tests { + use crate::threads::tests::recursive_function; + use crate::tokio_runtime::new_tokio_runtime; + + #[test] + fn test_stack_overflow() { + let rt = new_tokio_runtime("test_stack_overflow").build().unwrap(); + let error = rt + .block_on(async { + tokio::spawn(async { recursive_function(u32::MAX) }) + .await + .unwrap() + }) + .unwrap_err(); + assert!(error.to_string().contains("stack overflow"), "{error:?}"); + } + + #[test] + fn test_no_stack_overflow() { + let rt = new_tokio_runtime("test_stack_overflow").build().unwrap(); + let () = rt + .block_on(async { + tokio::spawn(async { recursive_function(1000) }) + .await + .unwrap() + }) + .unwrap(); + } +} diff --git a/app/buck2_wrapper_common/src/invocation_id.rs b/app/buck2_wrapper_common/src/invocation_id.rs index c888c5ce27648..63708df34af54 100644 --- a/app/buck2_wrapper_common/src/invocation_id.rs +++ b/app/buck2_wrapper_common/src/invocation_id.rs @@ -110,7 +110,7 @@ impl TraceId { } #[cfg(test)] -mod test { +mod tests { use super::*; #[test] diff --git a/app/buck2_wrapper_common/src/lib.rs b/app/buck2_wrapper_common/src/lib.rs index 2618f9f08edf5..8a9f7fa7d929d 100644 --- a/app/buck2_wrapper_common/src/lib.rs +++ b/app/buck2_wrapper_common/src/lib.rs @@ -94,13 +94,9 @@ pub fn killall(who_is_asking: WhoIsAsking, write: impl Fn(String)) -> bool { impl Printer { fn fmt_status(&mut self, process: &ProcessInfo, status: &str) -> String { - format!( - "{} {} ({}). {}", - status, - process.name, - process.pid, - shlex::join(process.cmd.iter().map(|s| s.as_str())), - ) + let cmd = shlex::try_join(process.cmd.iter().map(|s| s.as_str())) + .expect("Null byte unexpected"); + format!("{} {} ({}). {}", status, process.name, process.pid, cmd,) } fn failed_to_kill(&mut self, process: &ProcessInfo, error: anyhow::Error) { diff --git a/cfg/experimental/asserts.bzl b/cfg/experimental/asserts.bzl new file mode 100644 index 0000000000000..4114891b36251 --- /dev/null +++ b/cfg/experimental/asserts.bzl @@ -0,0 +1,31 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load(":types.bzl", "Modifier", "is_modifiers_match") + +def verify_normalized_target(target: str): + # Do some basic checks that target looks reasonably valid and normalized + # Targets should always be fully qualified to improve readability. + if "//" not in target or target.startswith("//") or ":" not in target: + fail( + "Must specify fully qualified target (ex. `cell//foo:bar`). Found `{}`".format( + target, + ), + ) + +def verify_normalized_modifier(modifier: Modifier): + if modifier == None: + pass + elif is_modifiers_match(modifier): + # TODO(scottcao): Add a test case for this once `bxl_test` supports testing failures + for key, sub_modifier in modifier.items(): + if key != "_type": + verify_normalized_modifier(sub_modifier) + elif isinstance(modifier, str): + verify_normalized_target(modifier) + else: + fail("Found unexpected modifier `{}` type `{}`".format(modifier, type(modifier))) diff --git a/cfg/experimental/cfg_constructor.bzl b/cfg/experimental/cfg_constructor.bzl index 2b9fe5b2fede6..79151033e4d54 100644 --- a/cfg/experimental/cfg_constructor.bzl +++ b/cfg/experimental/cfg_constructor.bzl @@ -35,7 +35,8 @@ def cfg_constructor_pre_constraint_analysis( package_modifiers: list[dict[str, typing.Any]] | None, # typing.Any is JSON form of modifier target_modifiers: list[Modifier] | None, - cli_modifiers: list[str]) -> (list[str], PostConstraintAnalysisParams): + cli_modifiers: list[str], + **_kwargs) -> (list[str], PostConstraintAnalysisParams): """ First stage of cfg constructor for modifiers. @@ -113,15 +114,16 @@ def cfg_constructor_post_constraint_analysis( for tagged_modifiers in params.merged_modifiers: for modifier in tagged_modifiers.modifiers: - constraint_setting_label, modifier_info = get_modifier_info( - refs = refs, - modifier = modifier, - location = tagged_modifiers.location, - constraint_setting_order = constraint_setting_order, - ) - modifier_infos = constraint_setting_to_modifier_infos.get(constraint_setting_label) or [] - modifier_infos.append(modifier_info) - constraint_setting_to_modifier_infos[constraint_setting_label] = modifier_infos + if modifier: + constraint_setting_label, modifier_info = get_modifier_info( + refs = refs, + modifier = modifier, + location = tagged_modifiers.location, + constraint_setting_order = constraint_setting_order, + ) + modifier_infos = constraint_setting_to_modifier_infos.get(constraint_setting_label) or [] + modifier_infos.append(modifier_info) + constraint_setting_to_modifier_infos[constraint_setting_label] = modifier_infos cfg = ConfigurationInfo( constraints = {}, diff --git a/cfg/experimental/common.bzl b/cfg/experimental/common.bzl index b282194849a53..4d4cb98628ddf 100644 --- a/cfg/experimental/common.bzl +++ b/cfg/experimental/common.bzl @@ -6,6 +6,7 @@ # of this source tree. load("@prelude//:asserts.bzl", "asserts") +load(":asserts.bzl", "verify_normalized_modifier") load( ":types.bzl", "Modifier", @@ -13,9 +14,10 @@ load( "ModifierInfo", "ModifierLocation", "ModifierPackageLocation", - "ModifierSelectInfo", "ModifierTargetLocation", + "ModifiersMatchInfo", "TaggedModifiers", + "is_modifiers_match", ) _TARGET_LOCATION_STR = "`metadata` attribute of target" @@ -31,36 +33,6 @@ def location_to_string(location: ModifierLocation) -> str: return _CLI_LOCATION_STR fail("Internal error. Unrecognized location type `{}` for location `{}`".format(type(location), location)) -def verify_normalized_target(target: str): - # Do some basic checks that target looks reasonably valid and normalized - # Targets should always be fully qualified to improve readability. - if "//" not in target or target.startswith("//") or ":" not in target: - fail( - "Must specify fully qualified target (ex. `cell//foo:bar`). Found `{}`".format( - target, - ), - ) - -def is_modifier_select(modifier: Modifier) -> bool: - if isinstance(modifier, str): - return False - if isinstance(modifier, dict): - if modifier["_type"] != "ModifierSelect": - fail("Found unknown dictionary `{}` for modifier".format(modifier)) - return True - fail("Modifier should either be a string or dict. Found `{}`".format(modifier)) - -def verify_normalized_modifier(modifier: Modifier): - if is_modifier_select(modifier): - # TODO(scottcao): Add a test case for this once `bxl_test` supports testing failures - for key, sub_modifier in modifier.items(): - if key != "_type": - verify_normalized_modifier(sub_modifier) - elif isinstance(modifier, str): - verify_normalized_target(modifier) - else: - fail("Found unexpected modifier `{}` type `{}`".format(modifier, type(modifier))) - def get_tagged_modifiers( modifiers: list[Modifier], location: ModifierLocation) -> TaggedModifiers: @@ -74,7 +46,7 @@ def get_tagged_modifiers( def get_constraint_setting(constraint_settings: dict[TargetLabel, None], modifier: Modifier, location: ModifierLocation) -> TargetLabel: if len(constraint_settings) == 0: - fail("`modifier_select` cannot be empty. Found empty `modifier_select` at `{}`".format(location_to_string(location))) + fail("`modifiers.match` cannot be empty. Found empty `modifiers.match` at `{}`".format(location_to_string(location))) if len(constraint_settings) > 1: fail( "A single modifier can only modify a single constraint setting.\n" + @@ -89,35 +61,43 @@ def get_modifier_info( refs: dict[str, ProviderCollection], modifier: Modifier, location: ModifierLocation, - constraint_setting_order: list[TargetLabel]) -> (TargetLabel, ModifierInfo): + constraint_setting_order: list[TargetLabel]) -> ((TargetLabel, ModifierInfo) | None): # Gets a modifier info from a modifier based on providers from `refs`. - if is_modifier_select(modifier): + if modifier == None: + return None + if is_modifiers_match(modifier): default = None - modifier_selector_info = [] + modifiers_match_info = [] constraint_settings = {} # Used like a set for key, sub_modifier in modifier.items(): if key == "DEFAULT": - default_constraint_setting, default = get_modifier_info(refs, sub_modifier, location, constraint_setting_order) - constraint_settings[default_constraint_setting] = None + if sub_modifier: + default_constraint_setting, default = get_modifier_info(refs, sub_modifier, location, constraint_setting_order) + constraint_settings[default_constraint_setting] = None + else: + default = None elif key != "_type": cfg_info = refs[key][ConfigurationInfo] for cfg_constraint_setting, cfg_constraint_value_info in cfg_info.constraints.items(): asserts.true( cfg_constraint_setting in constraint_setting_order, ( - "modifier_select `{}` from `{}` selects on `{}` of constraint_setting `{}`, which is now allowed. " + + "modifiers.match `{}` from `{}` selects on `{}` of constraint_setting `{}`, which is now allowed. " + "To select on this constraint, this constraint setting needs to be added to `buck2/cfg/experimental/cfg_constructor.bzl`" ).format(modifier, location_to_string(location), cfg_constraint_value_info.label, cfg_constraint_setting), ) - sub_constraint_setting, sub_modifier_info = get_modifier_info(refs, sub_modifier, location, constraint_setting_order) - constraint_settings[sub_constraint_setting] = None - modifier_selector_info.append((cfg_info, sub_modifier_info)) + if sub_modifier: + sub_constraint_setting, sub_modifier_info = get_modifier_info(refs, sub_modifier, location, constraint_setting_order) + constraint_settings[sub_constraint_setting] = None + else: + sub_modifier_info = None + modifiers_match_info.append((cfg_info, sub_modifier_info)) constraint_setting = get_constraint_setting(constraint_settings, modifier, location) - return constraint_setting, ModifierSelectInfo( + return constraint_setting, ModifiersMatchInfo( default = default, - selector = modifier_selector_info, + selector = modifiers_match_info, ) if isinstance(modifier, str): modifier_info = refs[modifier] @@ -141,7 +121,9 @@ def _is_subset(a: ConfigurationInfo, b: ConfigurationInfo) -> bool: def resolve_modifier(cfg: ConfigurationInfo, modifier: ModifierInfo) -> ConstraintValueInfo | None: # Resolve the modifier and return the constraint value to add to the configuration, if there is one - if isinstance(modifier, ModifierSelectInfo): + if modifier == None: + return None + if isinstance(modifier, ModifiersMatchInfo): for key, sub_modifier in modifier.selector: if _is_subset(key, cfg): # If constraints in key of the select are a subset of the constraints in the @@ -157,7 +139,9 @@ def resolve_modifier(cfg: ConfigurationInfo, modifier: ModifierInfo) -> Constrai def modifier_to_refs(modifier: Modifier, location: ModifierLocation) -> list[str]: # Obtain a list of targets to analyze from a modifier. refs = [] - if is_modifier_select(modifier): + if modifier == None: + pass + elif is_modifiers_match(modifier): for key, sub_modifier in modifier.items(): if key != "_type": if key != "DEFAULT": diff --git a/cfg/experimental/modifier_select.bzl b/cfg/experimental/modifiers.bzl similarity index 70% rename from cfg/experimental/modifier_select.bzl rename to cfg/experimental/modifiers.bzl index 7f06de0ebfeb7..b92a941aa28da 100644 --- a/cfg/experimental/modifier_select.bzl +++ b/cfg/experimental/modifiers.bzl @@ -6,28 +6,28 @@ # of this source tree. load("@fbsource//tools/build_defs/buck2:is_buck2.bzl", "is_buck2") -load(":common.bzl?v2_only", "verify_normalized_modifier", "verify_normalized_target") +load(":asserts.bzl?v2_only", "verify_normalized_modifier", "verify_normalized_target") load( ":types.bzl?v2_only", "Modifier", # @unused Used in type annotation - "ModifierSelect", + "ModifiersMatch", ) -def modifier_select( - selector: dict[str, Modifier | None]) -> ModifierSelect: +def _modifiers_match( + matcher: dict[str, Modifier]) -> ModifiersMatch: """ - A select operator for modifiers. A `modifier_select` specifies a way for a + A select operator for modifiers. A `modifiers.match` specifies a way for a modifier to be added based on an existing constraint in the configuration. - The `selector` is a dictionary that maps from a set of constraints to a + The `matcher` is a dictionary that maps from a set of constraints to a modifier. For example, suppose `cfg//os:linux` and `cfg//os:windows` are constraint values for OS and `cfg//compiler:clang` and `cfg//compiler:msvc` are constraint values - for compiler. The following `modifier_select` conditionally adds the msvc constraint + for compiler. The following `modifiers.match` conditionally adds the msvc constraint if the the windows constraint is matched or adds the clang constraint if the the linux constraint is matched. ``` - modifier_select({ + modifiers.match({ "cfg//os:windows": "cfg//compiler:msvc", "cfg//os:linux": "cfg//compiler:clang", "DEFAULT": None, @@ -37,17 +37,17 @@ def modifier_select( then the modifier specified by DEFAULT will be used. If None is specified, then a modifier will not be added. - `modifier_select`s can be stacked. For example, + `modifiers.match`s can be stacked. For example, suppose this modifier is specified in fbcode/PACKAGE ``` - modifier = modifier_select({ + modifier = modifiers.match({ "cfg//os:windows": "cfg//compiler:msvc", "DEFAULT": None, }) ``` Suppose this modifier is specified in fbcode/project/PACKAGE ``` - modifier = modifier_select({ + modifier = modifiers.match({ "cfg//os:linux": "cfg//compiler:clang", "DEFAULT": None, }) @@ -55,9 +55,9 @@ def modifier_select( For any target covered by fbcode/project/PACKAGE, this is equivalent to one modifier in that specifies ``` - modifier_select({ + modifiers.match({ "cfg//os:windows": "cfg//compiler:msvc", - "DEFAULT": modifier_select({ + "DEFAULT": modifiers.match({ "DEFAULT": None, "cfg//os:linux": "cfg//compiler:clang", }) @@ -68,10 +68,16 @@ def modifier_select( if not is_buck2(): return {} - for key, sub_modifier in selector.items(): + for key, sub_modifier in matcher.items(): if key != "DEFAULT": verify_normalized_target(key) verify_normalized_modifier(sub_modifier) - selector["_type"] = "ModifierSelect" - return selector + matcher["_type"] = "ModifiersMatch" + return matcher + +modifiers = struct( + # modifiers.match is deprecated for modifiers.conditional + match = _modifiers_match, + conditional = _modifiers_match, +) diff --git a/cfg/experimental/order.bzl b/cfg/experimental/order.bzl index 1e980bf48cef5..5e1501ad35fc7 100644 --- a/cfg/experimental/order.bzl +++ b/cfg/experimental/order.bzl @@ -5,9 +5,9 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -# List of constraint settings that can be selected on via `modifier_select`. +# List of constraint settings that can be selected on via `modifiers_match`. # Modifiers for these constraints are resolved first before other modifiers, -# if they exist. `modifier_select` keying on other constraint settings will +# if they exist. `modifiers_match` keying on other constraint settings will # fail # TODO(scottcao): Find a better place to set this so that OSS users can add # their own constraints. diff --git a/cfg/experimental/set_cfg_modifiers.bzl b/cfg/experimental/set_cfg_modifiers.bzl index 6cc601a01ab2a..cec6bd7d3ad00 100644 --- a/cfg/experimental/set_cfg_modifiers.bzl +++ b/cfg/experimental/set_cfg_modifiers.bzl @@ -27,6 +27,15 @@ def set_cfg_modifiers(modifiers: list[Modifier]): if not is_buck2(): return + # Make this buck1-proof + call_stack_frame = getattr(native, "call_stack_frame", None) + + # To ensure that modifiers set in PACKAGE files are easily codemoddable + # We want to enforce that `set_cfg_modifiers` is only invokable from a PACKAGE file and not a bzl file + module_path = call_stack_frame(1).module_path + if not module_path.endswith("/PACKAGE") and module_path != "PACKAGE": + fail("set_cfg_modifiers is only allowed to be used from PACKAGE files, not a bzl file") + # Make this buck1-proof write_package_value = getattr(native, "write_package_value", None) read_parent_package_value = getattr(native, "read_parent_package_value", None) diff --git a/cfg/experimental/types.bzl b/cfg/experimental/types.bzl index e299dff4a9657..1c9f55ae4c0a1 100644 --- a/cfg/experimental/types.bzl +++ b/cfg/experimental/types.bzl @@ -24,9 +24,9 @@ ModifierLocation = ModifierPackageLocation | ModifierTargetLocation | ModifierCl # Modifier types as how they appear to the user via `set_cfg_modifier` or `cfg_modifier` function. -ModifierSelect = dict[str, typing.Any] +ModifiersMatch = dict[str, typing.Any] -Modifier = str | ModifierSelect +Modifier = str | ModifiersMatch | None TaggedModifiers = record( modifiers = list[Modifier], @@ -38,10 +38,19 @@ TaggedModifiers = record( # An "Info" is added to the type name to denote post-constraint-analysis version of the # modifier type. -ModifierSelectInfo = record( +ModifiersMatchInfo = record( # should be list[(ConfigurationInfo, "ModifierInfo")] once recursive types are supported selector = list[(ConfigurationInfo, typing.Any)], default = typing.Any, # should be "ModifierInfo" | None with recursive types ) -ModifierInfo = ConstraintValueInfo | ModifierSelectInfo +ModifierInfo = ConstraintValueInfo | ModifiersMatchInfo | None + +def is_modifiers_match(modifier: Modifier) -> bool: + if modifier == None or isinstance(modifier, str): + return False + if isinstance(modifier, dict): + if modifier["_type"] != "ModifiersMatch": + fail("Found unknown dictionary `{}` for modifier".format(modifier)) + return True + fail("Modifier should either be None, a string, or dict. Found `{}`".format(modifier)) diff --git a/dice/README.md b/dice/README.md index 61d532dda479f..8929e27b9ef09 100644 --- a/dice/README.md +++ b/dice/README.md @@ -5,8 +5,10 @@ incremental computation engine that supports parallel computation. ## Documentation -For detailed documentation, see the docs in [dice/docs/index.md](dice/docs/index.md) +For detailed documentation, see the docs in +[dice/docs/index.md](dice/docs/index.md) ## License -DICE is both MIT and Apache License, Version 2.0 licensed, as found in the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. +DICE is both MIT and Apache License, Version 2.0 licensed, as found in the +[LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. diff --git a/dice/dice/BUCK b/dice/dice/BUCK index 88b270b6d00ec..0a270f5f898b3 100644 --- a/dice/dice/BUCK +++ b/dice/dice/BUCK @@ -30,6 +30,7 @@ rust_library( "fbsource//third-party/rust:scopeguard", "fbsource//third-party/rust:serde", "fbsource//third-party/rust:slab", + "fbsource//third-party/rust:static_assertions", "fbsource//third-party/rust:take_mut", "fbsource//third-party/rust:thiserror", "fbsource//third-party/rust:tokio", diff --git a/dice/dice/Cargo.toml b/dice/dice/Cargo.toml index ca25cb0f75d7c..aba7b23e6c6f0 100644 --- a/dice/dice/Cargo.toml +++ b/dice/dice/Cargo.toml @@ -12,7 +12,7 @@ anymap = "0.12.1" async-trait = "0.1.24" buck2_futures = { path = "../../app/buck2_futures" } cmp_any = { workspace = true } -dashmap = "4.0.2" +dashmap = "5.5.3" derivative = { workspace = true } derive_more = "0.99.3" dupe = { workspace = true } @@ -29,6 +29,7 @@ serde = { version = "1.0", features = ["derive"] } slab = "0.4.7" # @oss-disable: sorted_vector_map.path = "../../../common/rust/shed/sorted_vector_map" sorted_vector_map.version = "0.1" +static_assertions = { workspace = true } take_mut = { workspace = true } thiserror = "1.0.36" tokio = { version = "1.5", features = ["full"] } diff --git a/dice/dice/docs/api.md b/dice/dice/docs/api.md index 1f0d7010d342f..ee3e979b3f943 100644 --- a/dice/dice/docs/api.md +++ b/dice/dice/docs/api.md @@ -2,22 +2,28 @@ Everything starts at the base struct `Dice`. -From it, you can get the `TransactionUpdater` where you can report the changes to your computation graph you expect -to see this time. You can dirty nodes to be recomputed via `TransactionUpdater::changed`, -or report new [Injected Values](writing_computations.md#injected-keys) via `TransactionUpdater::changed_to`. -These changes recorded will not be seen by anyone until you commit them via `TransactionUpdater::commit`, which returns -you an instance of `DiceTransaction` containing all changes recorded up until this moment (but not any future changes). +From it, you can get the `TransactionUpdater` where you can report the changes +to your computation graph you expect to see this time. You can dirty nodes to be +recomputed via `TransactionUpdater::changed`, or report new +[Injected Values](writing_computations.md#injected-keys) via +`TransactionUpdater::changed_to`. These changes recorded will not be seen by +anyone until you commit them via `TransactionUpdater::commit`, which returns you +an instance of `DiceTransaction` containing all changes recorded up until this +moment (but not any future changes). -`DiceTransaction` instances that have the same set of reported changes are considered to be of the same state. Computations -can be requested from them concurrently, and work will be shared. -However, concurrent requests to `DiceTransactions` with different states are NOT supported. -DICE currently does NOT enforce this rule. Please be aware. +`DiceTransaction` instances that have the same set of reported changes are +considered to be of the same state. Computations can be requested from them +concurrently, and work will be shared. However, concurrent requests to +`DiceTransactions` with different states are NOT supported. DICE currently does +NOT enforce this rule. Please be aware. +The function `DiceComputations::compute(Key)` is used to compute a specific Key, +returning a future to the result. `DiceTransaction` derefs into a +`DiceComputations` such computations can be called directly on the transaction. -The function `DiceComputations::compute(Key)` is used to compute a specific Key, returning a future to the result. -`DiceTransaction` derefs into a `DiceComputations` such computations can be called directly on the transaction. +To make code look natural, we often employ the pattern of writing computation +traits as follows: -To make code look natural, we often employ the pattern of writing computation traits as follows: ```rust #[async_trait] trait MyComputationTrait { @@ -38,7 +44,9 @@ impl MyComputationTrait for DiceComputations { } ``` -This will let you write the more natural code as follows instead of having to explicitly create and refer to Keys everywhere in the code base: +This will let you write the more natural code as follows instead of having to +explicitly create and refer to Keys everywhere in the code base: + ```rust use MyComputationTrait; @@ -52,6 +60,7 @@ async fn main() { } ``` -You can group computations in traits however you wish, with whatever Key types you desire. In buck2, we tend to group -related computations together (i.e all parsing computations in one trait, or all action computations in one trait) to +You can group computations in traits however you wish, with whatever Key types +you desire. In buck2, we tend to group related computations together (i.e all +parsing computations in one trait, or all action computations in one trait) to better organize our modules and limit the number of dependencies pulled in. diff --git a/dice/dice/docs/cancellations.md b/dice/dice/docs/cancellations.md index 173759f256cb9..59d318cb0966f 100644 --- a/dice/dice/docs/cancellations.md +++ b/dice/dice/docs/cancellations.md @@ -1,22 +1,28 @@ # Cancellations -DICE supports cancellations of computations you have requested. -Since each computation is returned as a future, dropping the future will notify DICE that the computation is no longer -needed, and can be cancelled if appropriate. Note that dropping the future does not guarantee that computation is -canceled because of multi-tenancy, where if any other request depends on the same computation, the computation will -continue to run to completion for the other request. +DICE supports cancellations of computations you have requested. Since each +computation is returned as a future, dropping the future will notify DICE that +the computation is no longer needed, and can be cancelled if appropriate. Note +that dropping the future does not guarantee that computation is canceled because +of multi-tenancy, where if any other request depends on the same computation, +the computation will continue to run to completion for the other request. ## How It Works -DICE tracks all currently running computations in a map of [`WeakShared`](https://docs.rs/futures/0.3.17/futures/future/struct.WeakShared.html) -`DiceTask`s. This core map is what allows concurrent requests share work when they request for the same computation. -When a request requires a computation that is currently running in the map, it will attempt to acquire a [`Shared`](https://docs.rs/futures/0.3.17/futures/future/struct.Shared.html) -version of the `WeakShared`. If the `WeakShared` was already dropped, acquiring a `Shared` will fail, causing the -request to spawn a new `DiceTask`, holding onto a `Shared` and inserting its corresponding `WeakShared` into the map. +DICE tracks all currently running computations in a map of +[`WeakShared`](https://docs.rs/futures/0.3.17/futures/future/struct.WeakShared.html) +`DiceTask`s. This core map is what allows concurrent requests share work when +they request for the same computation. When a request requires a computation +that is currently running in the map, it will attempt to acquire a +[`Shared`](https://docs.rs/futures/0.3.17/futures/future/struct.Shared.html) +version of the `WeakShared`. If the `WeakShared` was already dropped, acquiring +a `Shared` will fail, causing the request to spawn a new `DiceTask`, holding +onto a `Shared` and inserting its corresponding `WeakShared` into the map. -When a request is canceled by dropping its future, the `Shared` it holds will be dropped. -When there are no strong references (i.e `Shared` versions) of the `WeakShared`, the `DiceTask` will be dropped, -which triggers the spawned task to be aborted. -By having only active requests hold onto the `Shared`, and the map itself holding only a `WeakShared`, we can guarantee -that the futures are never canceled if there is a request actively depending on it, and that the future will be canceled -once there are no active requests for it. +When a request is canceled by dropping its future, the `Shared` it holds will be +dropped. When there are no strong references (i.e `Shared` versions) of the +`WeakShared`, the `DiceTask` will be dropped, which triggers the spawned task to +be aborted. By having only active requests hold onto the `Shared`, and the map +itself holding only a `WeakShared`, we can guarantee that the futures are never +canceled if there is a request actively depending on it, and that the future +will be canceled once there are no active requests for it. diff --git a/dice/dice/docs/incrementality.md b/dice/dice/docs/incrementality.md index 2a900ca1cbfea..281dae815648d 100644 --- a/dice/dice/docs/incrementality.md +++ b/dice/dice/docs/incrementality.md @@ -1,19 +1,31 @@ # Incrementality -Incrementality is the idea that given any changes of a collection of interdependent computations, -only the changed portions of the computation are recomputed. To record changes and to discover the changed portion, -portions of the dependency graph needs to be traversed to discover the changes. +Incrementality is the idea that given any changes of a collection of +interdependent computations, only the changed portions of the computation are +recomputed. To record changes and to discover the changed portion, portions of +the dependency graph needs to be traversed to discover the changes. -DICE tracks the reverse dependencies (rdeps) of computations to achieve O(invalidated subset) traversals and O(changed subset) recomputations for a given request. -* 'invalidated subset' is the set of all possibly invalidated nodes intersected with the set of nodes that might be needed for the request regardless of whether a node is cached or not. -* 'changed subset' is the set nodes whose values changed intersected with the set of nodes that might be needed for the request regardless of whether a node is cached or not. +DICE tracks the reverse dependencies (rdeps) of computations to achieve +O(invalidated subset) traversals and O(changed subset) recomputations for a +given request. + +- 'invalidated subset' is the set of all possibly invalidated nodes intersected + with the set of nodes that might be needed for the request regardless of + whether a node is cached or not. +- 'changed subset' is the set nodes whose values changed intersected with the + set of nodes that might be needed for the request regardless of whether a node + is cached or not. This allows DICE to minimize the amount of work performed for each new request. # Multi-versioning -DICE supports a multi-commit transaction model, where computations/requests that is currently running do not see newly -committed changes. We do not yet support running transactions at different versions concurrently nor storing multiple versions -of nodes. + +DICE supports a multi-commit transaction model, where computations/requests that +is currently running do not see newly committed changes. We do not yet support +running transactions at different versions concurrently nor storing multiple +versions of nodes. # Details -Details of the incrementality algorithm can be found in the [PDF](DiceIncrementalityAlgorithms.pdf). + +Details of the incrementality algorithm can be found in the +[PDF](DiceIncrementalityAlgorithms.pdf). diff --git a/dice/dice/docs/index.md b/dice/dice/docs/index.md index 5d3979a464177..46897b9bd66a7 100644 --- a/dice/dice/docs/index.md +++ b/dice/dice/docs/index.md @@ -1,29 +1,38 @@ # DICE -DICE is a dynamic incremental computation engine that supports parallel computation, inspired by -[Adapton](https://docs.rs/adapton/latest/adapton/) and [Salsa](https://github.com/salsa-rs/salsa), +DICE is a dynamic incremental computation engine that supports parallel +computation, inspired by [Adapton](https://docs.rs/adapton/latest/adapton/) and +[Salsa](https://github.com/salsa-rs/salsa), -DICE is the core computation engine that powers the incremental graph transformations of [buck2](https://github.com/facebook/buck2). -It is intended to offer a generic computation API that can be used beyond just Buck, so that any kind of incremental computation can run on DICE. -All computations are executed in parallel on DICE via tokio executors. Duplicate requests to the same computations are deduplicated. +DICE is the core computation engine that powers the incremental graph +transformations of [buck2](https://github.com/facebook/buck2). It is intended to +offer a generic computation API that can be used beyond just Buck, so that any +kind of incremental computation can run on DICE. All computations are executed +in parallel on DICE via tokio executors. Duplicate requests to the same +computations are deduplicated. DICE is currently still experimental and largely being rewritten. ## Features + - [Incrementality](incrementality.md) - Incrementality behaviour of DICE - [Parallelism](parallelism.md) - Parallelism and Behaviour of Computations -- [Cancellations](cancellations.md) - Cancelling of a currently running computation +- [Cancellations](cancellations.md) - Cancelling of a currently running + computation - [Transient Errors](transients.md) - Transient Error Handling - [Projections](projections.md) - Projection Computations - Cycle Detection // TODO ## Using DICE + - [Basic API](api.md) - How to use DICE -- [Writing Computations](writing_computations.md) - How to write computations that are incremental +- [Writing Computations](writing_computations.md) - How to write computations + that are incremental ## Benchmarking DICE -// TODO +// TODO ## Debugging the Graph + // TODO diff --git a/dice/dice/docs/parallelism.md b/dice/dice/docs/parallelism.md index 14ad4c2af60c3..edaa269a90b35 100644 --- a/dice/dice/docs/parallelism.md +++ b/dice/dice/docs/parallelism.md @@ -1,9 +1,12 @@ # Parallelism and Computation Behaviour -Every computation in DICE is automatically spawned asynchronously in Rust via tokio to be computed in parallel. -They behave like standard Rust futures, suspending when users `await` dependent computations and resuming when the +Every computation in DICE is automatically spawned asynchronously in Rust via +tokio to be computed in parallel. They behave like standard Rust futures, +suspending when users `await` dependent computations and resuming when the dependent futures are ready. -The same identical computation is always deduplicated, so concurrent requests to the same exact key will only be -executed once if not cached. Additionally, for normal computations, we guarantee that the same instance of the computed -value is returned to all requests if the value is considered equal based on the Key's equality. +The same identical computation is always deduplicated, so concurrent requests to +the same exact key will only be executed once if not cached. Additionally, for +normal computations, we guarantee that the same instance of the computed value +is returned to all requests if the value is considered equal based on the Key's +equality. diff --git a/dice/dice/docs/projections.md b/dice/dice/docs/projections.md index 3837ef8764a6c..4cb175756da65 100644 --- a/dice/dice/docs/projections.md +++ b/dice/dice/docs/projections.md @@ -1,19 +1,25 @@ # Projection Computations DICE supports a special type of synchronous computation called "Projections". -These are synchronous computations that are derived from the result of a larger parallel async computation. - -This allows computations to depend on only "portions" of the result of another computation, allowing the parent computation -to be resurrected more often and not need be recomputed if only the unused portions of the dependent result changes. -For example, you may have a computation that retrieves and parses JSON. Now you have an expensive computation that -requires a single value of the JSON. Rather than depending on the entirety of the JSON and having to rerun the expensive -computation whenever any of the JSON value changes, you can write a new Projection Computation that provides access to -specific values from the JSON. Now you can have the expensive computation depend on the projection, which will avoid needing -to rerun the expensive computation unless that specific projected value changes. +These are synchronous computations that are derived from the result of a larger +parallel async computation. + +This allows computations to depend on only "portions" of the result of another +computation, allowing the parent computation to be resurrected more often and +not need be recomputed if only the unused portions of the dependent result +changes. For example, you may have a computation that retrieves and parses JSON. +Now you have an expensive computation that requires a single value of the JSON. +Rather than depending on the entirety of the JSON and having to rerun the +expensive computation whenever any of the JSON value changes, you can write a +new Projection Computation that provides access to specific values from the +JSON. Now you can have the expensive computation depend on the projection, which +will avoid needing to rerun the expensive computation unless that specific +projected value changes. ## API -To create a Projection Computation, create a struct and implement `ProjectionKey`. +To create a Projection Computation, create a struct and implement +`ProjectionKey`. ```rust struct MyProjection; @@ -35,13 +41,16 @@ impl dice::api::ProjectionKey for MyProjection { } } ``` -The `BaseComputeKey` is the async computation for which the projected values are based off of. +The `BaseComputeKey` is the async computation for which the projected values are +based off of. -To request the projection, you must compute the base via `DiceComputations::compute_opaque(Key)` which returns a `OpaqueValue`. -Then, request the projection via `OpaqueValue::projection(ProjectionKey)`. +To request the projection, you must compute the base via +`DiceComputations::compute_opaque(Key)` which returns a `OpaqueValue`. Then, +request the projection via `OpaqueValue::projection(ProjectionKey)`. -Similar to normal keys, buck2 often hides the keys to make code look more natural via traits. +Similar to normal keys, buck2 often hides the keys to make code look more +natural via traits. ```rust #[async_trait] @@ -76,7 +85,9 @@ impl SyncProjectionTrait for OpaqueValue { } ``` -This will let you write the more natural code as follows instead of having to explicitly create and refer to Keys everywhere in the code base: +This will let you write the more natural code as follows instead of having to +explicitly create and refer to Keys everywhere in the code base: + ```rust use MyComputationTrait; use SyncProjectionTrait; diff --git a/dice/dice/docs/transients.md b/dice/dice/docs/transients.md index 0d6422aa0261a..ac313731aab3b 100644 --- a/dice/dice/docs/transients.md +++ b/dice/dice/docs/transients.md @@ -1,12 +1,15 @@ # Transient Errors -DICE has a concept of "transient" errors, which are errors that are non-deterministic and should be retried instead of -cached. -These are indicated by `Key::validity(Key::Value)` returning `false`. +DICE has a concept of "transient" errors, which are errors that are +non-deterministic and should be retried instead of cached. These are indicated +by `Key::validity(Key::Value)` returning `false`. -When DICE encounters a "transient value", the value is reused for the ongoing computation transaction. That is, all -active requests of the same transaction will see the same instance of the transient value. However, this value will -not be cached such that upon obtaining a new transaction with or without committing any changes to the graph, the value -will be recomputed (once). If the recompute still results in a transient, then the value is still not cached and the -same behaviour occurs on the next fresh transaction. If the recompute results in a non-transient value, then the value -will be cached, and the next transaction will reuse the cached value if there are no changes that invalidate the value. +When DICE encounters a "transient value", the value is reused for the ongoing +computation transaction. That is, all active requests of the same transaction +will see the same instance of the transient value. However, this value will not +be cached such that upon obtaining a new transaction with or without committing +any changes to the graph, the value will be recomputed (once). If the recompute +still results in a transient, then the value is still not cached and the same +behaviour occurs on the next fresh transaction. If the recompute results in a +non-transient value, then the value will be cached, and the next transaction +will reuse the cached value if there are no changes that invalidate the value. diff --git a/dice/dice/docs/writing_computations.md b/dice/dice/docs/writing_computations.md index 0fbc6b3885828..61b1ffbf12d36 100644 --- a/dice/dice/docs/writing_computations.md +++ b/dice/dice/docs/writing_computations.md @@ -1,8 +1,10 @@ # Writing a Computation + Computations are written by declaring a struct that implements the `Key` trait. -In this trait, you will declare a `compute` function that is the calculation to perform when not cached. -This method will receive a context `DiceComputations`, which is where you can request for further keys' values. These -keys will be recorded as dependencies of the current computation. +In this trait, you will declare a `compute` function that is the calculation to +perform when not cached. This method will receive a context `DiceComputations`, +which is where you can request for further keys' values. These keys will be +recorded as dependencies of the current computation. ```rust #[derive(Allocative, Clone, Debug, Display, Eq, PartialEq, Hash)] @@ -24,17 +26,21 @@ impl dice::api::Key for MyKey { } ``` -Additionally, there are methods `equality` and `validity` that one can implement for the `Key` trait to configure -the behaviour of [transients](transients.md) and equals for the output of the computation. Equals allows DICE to resurrect -nodes that depends on values that were invalidated, but end up recomputing to the "same" value. +Additionally, there are methods `equality` and `validity` that one can implement +for the `Key` trait to configure the behaviour of [transients](transients.md) +and equals for the output of the computation. Equals allows DICE to resurrect +nodes that depends on values that were invalidated, but end up recomputing to +the "same" value. ## Injected Keys -Injected Keys are a special type of Keys that are NOT computed. They must have their value explicitly set when informing -DICE of updated values via `TransactionUpdater::changed_to`, and all future requests will yield such value until updated -again. -Computations are written by declaring a struct that implements the `InjectedKey` trait. -They have no compute function, but offers `equality` as well. +Injected Keys are a special type of Keys that are NOT computed. They must have +their value explicitly set when informing DICE of updated values via +`TransactionUpdater::changed_to`, and all future requests will yield such value +until updated again. + +Computations are written by declaring a struct that implements the `InjectedKey` +trait. They have no compute function, but offers `equality` as well. ```rust struct MyInjectedKey; diff --git a/dice/dice/src/api/computations.rs b/dice/dice/src/api/computations.rs index 973fd00d0ab80..f9c32a2b62fb5 100644 --- a/dice/dice/src/api/computations.rs +++ b/dice/dice/src/api/computations.rs @@ -142,6 +142,17 @@ impl<'a> DerefMut for DiceComputationsParallel<'a> { } } +// This assertion assures we don't unknowingly regress the size of this critical future. +// TODO(cjhopman): We should be able to wrap this in a convenient assertion macro. +#[allow(unused, clippy::diverging_sub_expression)] +fn _assert_dice_compute_future_sizes() { + let ctx: DiceComputations = panic!(); + let k: K = panic!(); + let v = ctx.compute(&k); + let e = [0u8; 640 / 8]; + static_assertions::assert_eq_size_ptr!(&v, &e); +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/dice/dice/src/api/data.rs b/dice/dice/src/api/data.rs index f00d5a3e12777..b85faaf386c4c 100644 --- a/dice/dice/src/api/data.rs +++ b/dice/dice/src/api/data.rs @@ -19,11 +19,11 @@ //! use crate::dice::DiceData; //! //! pub trait HasData { -//! fn my_data(&self) -> usize; +//! fn my_data(&self) -> usize; //! -//! fn other_data(&self) -> &String; +//! fn other_data(&self) -> &String; //! -//! fn set_multi(&mut self, i: usize, s: String); +//! fn set_multi(&mut self, i: usize, s: String); //! } //! //! struct HasDataContainer(usize, String); @@ -47,9 +47,7 @@ //! //! assert_eq!(data.other_data(), &"foo".to_string()); //! assert_eq!(data.my_data(), 1); -//! //! ``` -//! use std::collections::BTreeSet; diff --git a/dice/dice/src/impls/core/graph/nodes.rs b/dice/dice/src/impls/core/graph/nodes.rs index 63ea3b6987e47..5a12325f6e9b2 100644 --- a/dice/dice/src/impls/core/graph/nodes.rs +++ b/dice/dice/src/impls/core/graph/nodes.rs @@ -16,7 +16,6 @@ //! The 'VersionedCache' will track dependency edges and use computed version //! number for each cache entry and a global version counter to determine //! up-to-date-ness of cache entries. -//! use allocative::Allocative; use dupe::Dupe; diff --git a/dice/dice/src/impls/core/graph/storage.rs b/dice/dice/src/impls/core/graph/storage.rs index df329cddc3513..28fc19abc910c 100644 --- a/dice/dice/src/impls/core/graph/storage.rs +++ b/dice/dice/src/impls/core/graph/storage.rs @@ -16,7 +16,6 @@ //! The 'VersionedCache' will track dependency edges and use computed version //! number for each cache entry and a global version counter to determine //! up-to-date-ness of cache entries. -//! use std::cmp; use std::ops::Bound; diff --git a/dice/dice/src/impls/incremental/mod.rs b/dice/dice/src/impls/incremental/mod.rs index d75958898654b..a7ace7907abe2 100644 --- a/dice/dice/src/impls/incremental/mod.rs +++ b/dice/dice/src/impls/incremental/mod.rs @@ -12,7 +12,6 @@ //! //! This is responsible for performing incremental caching and invalidations //! with multiple versions in-flight at the same time. -//! use std::borrow::Cow; use std::fmt::Debug; diff --git a/dice/dice/src/impls/incremental/tests.rs b/dice/dice/src/impls/incremental/tests.rs index 5a1dddbf588c6..2997a248c0109 100644 --- a/dice/dice/src/impls/incremental/tests.rs +++ b/dice/dice/src/impls/incremental/tests.rs @@ -12,7 +12,6 @@ //! //! This is responsible for performing incremental caching and invalidations //! with multiple versions in-flight at the same time. -//! use std::fmt::Debug; use std::hash::Hash; diff --git a/dice/dice/src/legacy/dice_futures/future_handle.rs b/dice/dice/src/legacy/dice_futures/future_handle.rs index bb325085ba385..dd4277a4efe17 100644 --- a/dice/dice/src/legacy/dice_futures/future_handle.rs +++ b/dice/dice/src/legacy/dice_futures/future_handle.rs @@ -9,7 +9,6 @@ //! The future that is spawned and managed by DICE. This is a single computation unit that is //! shareable across different computation units. -//! use allocative::Allocative; use buck2_futures::spawn::CompletionObserver; use buck2_futures::spawn::WeakFutureError; diff --git a/dice/dice/src/legacy/incremental/graph/dependencies.rs b/dice/dice/src/legacy/incremental/graph/dependencies.rs index 043f386baaa7d..d9c0e25257c0c 100644 --- a/dice/dice/src/legacy/incremental/graph/dependencies.rs +++ b/dice/dice/src/legacy/incremental/graph/dependencies.rs @@ -159,7 +159,9 @@ impl Eq for Rdep {} impl Hash for Rdep { fn hash(&self, state: &mut H) { - self.0.upgrade().map(|p| Arc::as_ptr(&p)).hash(state) + // Cast to *const () to ignore the metadata associated with self.0 + // and conform to the PartialEq implementation above. + (Weak::as_ptr(&self.0) as *const ()).hash(state) } } diff --git a/dice/dice/src/legacy/incremental/graph/mod.rs b/dice/dice/src/legacy/incremental/graph/mod.rs index 6caf1962358e8..82380a8b7535a 100644 --- a/dice/dice/src/legacy/incremental/graph/mod.rs +++ b/dice/dice/src/legacy/incremental/graph/mod.rs @@ -16,7 +16,6 @@ //! The 'VersionedCache' will track dependency edges and use computed version //! number for each cache entry and a global version counter to determine //! up-to-date-ness of cache entries. -//! pub(crate) mod dependencies; pub(crate) mod storage_properties; diff --git a/dice/dice/src/legacy/incremental/mod.rs b/dice/dice/src/legacy/incremental/mod.rs index b7650af72374c..bdd64de9675cc 100644 --- a/dice/dice/src/legacy/incremental/mod.rs +++ b/dice/dice/src/legacy/incremental/mod.rs @@ -12,7 +12,6 @@ //! //! This is responsible for performing incremental caching and invalidations //! with multiple versions in-flight at the same time. -//! pub(crate) mod dep_trackers; pub(crate) mod evaluator; @@ -1372,21 +1371,20 @@ mod tests { .val()); assert_eq!(t, 3); + let node2 = engine + .get_cached(2, VersionNumber::new(1), MinorVersion::testing_new(0)) + .into_dyn(); + let node3 = engine + .get_cached(3, VersionNumber::new(1), MinorVersion::testing_new(0)) + .into_dyn(); let mut expected = HashSet::from_iter([ - Arc::as_ptr( - &engine - .get_cached(2, VersionNumber::new(1), MinorVersion::testing_new(0)) - .into_dyn(), - ), - Arc::as_ptr( - &engine - .get_cached(3, VersionNumber::new(1), MinorVersion::testing_new(0)) - .into_dyn(), - ), + (node2.key(), node2.is_valid()), + (node3.key(), node3.is_valid()), ]); for rdep in node.read_meta().rdeps.rdeps().rdeps.iter() { + let node = rdep.0.0.upgrade().unwrap(); assert!( - expected.remove(&Arc::as_ptr(&rdep.0.0.upgrade().unwrap())), + expected.remove(&(node.key(), node.is_valid())), "Extra rdeps" ); } diff --git a/dice/dice/src/legacy/tests/mod.rs b/dice/dice/src/legacy/tests/mod.rs index 56b90682d74aa..8f197b0605a81 100644 --- a/dice/dice/src/legacy/tests/mod.rs +++ b/dice/dice/src/legacy/tests/mod.rs @@ -287,18 +287,18 @@ fn ctx_tracks_rdeps_properly() -> anyhow::Result<()> { let mut expected_deps = ((k + 1)..6) .map(K) .map(|k| { - Arc::as_ptr( - &dice - .find_cache::() - .get_cached(k, VersionNumber::new(0), *vg.minor_version_guard) - .into_dyn(), - ) + let node = dice + .find_cache::() + .get_cached(k, VersionNumber::new(0), *vg.minor_version_guard) + .into_dyn(); + (node.key(), node.is_valid()) }) .collect::>(); for rdep in cached.read_meta().rdeps.rdeps().rdeps.iter() { + let node = rdep.0.0.upgrade().unwrap(); assert!( - expected_deps.remove(&Arc::as_ptr(&rdep.0.0.upgrade().unwrap())), + expected_deps.remove(&(node.key(), node.is_valid())), "Extra rdeps" ) } diff --git a/dice/oss/CHANGELOG.md b/dice/oss/CHANGELOG.md index 8f51278a58e3e..1834602de1536 100644 --- a/dice/oss/CHANGELOG.md +++ b/dice/oss/CHANGELOG.md @@ -1,3 +1,3 @@ # DICE -* Initial version. +- Initial version. diff --git a/dice/oss/CONTRIBUTING.md b/dice/oss/CONTRIBUTING.md index c7042cbe51c3b..3726e31cc8fcc 100644 --- a/dice/oss/CONTRIBUTING.md +++ b/dice/oss/CONTRIBUTING.md @@ -1,12 +1,13 @@ # Contributing to DICE -We want to make contributing to this project as easy and transparent as possible. +We want to make contributing to this project as easy and transparent as +possible. ## Our Development Process -DICE is currently developed in Facebook's internal repositories and then exported -out to GitHub by a Facebook team member; however, we invite you to submit pull -requests as described below. +DICE is currently developed in Facebook's internal repositories and then +exported out to GitHub by a Facebook team member; however, we invite you to +submit pull requests as described below. ## Pull Requests @@ -41,6 +42,6 @@ Follow the automatic `rust fmt` configuration. ## License -By contributing to DICE, you agree that your contributions will be -licensed under both the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) +By contributing to DICE, you agree that your contributions will be licensed +under both the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files in the root directory of this source tree. diff --git a/dice/oss/Cargo.toml b/dice/oss/Cargo.toml index f66d7c41ea092..761a920ed4105 100644 --- a/dice/oss/Cargo.toml +++ b/dice/oss/Cargo.toml @@ -12,6 +12,6 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/facebook/buck2" [workspace.dependencies] -allocative = { version = "0.3.1" } +allocative = { version = "0.3.2" } gazebo = { version = "0.8.1", path = "gazebo/gazebo" } gazebo_derive = { version = "0.8.1", path = "gazebo/gazebo_derive" } diff --git a/docs/concepts/buck_out.md b/docs/concepts/buck_out.md new file mode 100644 index 0000000000000..5a76dd36f044d --- /dev/null +++ b/docs/concepts/buck_out.md @@ -0,0 +1,22 @@ +--- +id: buck_out +title: buck-out +--- + +# buck-out + +Buck2 stores build artifacts in a directory named `buck-out` in the root of your +[project](glossary.md#project). You should not make assumptions about where +Buck2 places your build artifacts within the directory structure beneath +`buck-out` as these locations depend on Buck2's implementation and could +potentially change over time. Instead, to obtain the location of the build +artifact for a particular target, you can use one of the `--show-*-output` +options with the [`buck2 build`](../../users/commands/build) or +[`buck2 targets`](../../users/commands/targets) commands, most commonly +`--show-output`. For the full list of ways to show the output location, you can +run `buck2 build --help` or `buck2 targets --help`. + +``` +buck2 targets --show-output +buck2 build --show-output +``` diff --git a/docs/concepts/buckconfig.md b/docs/concepts/buckconfig.md index 2a1990ca44950..d82cf0c97a29d 100644 --- a/docs/concepts/buckconfig.md +++ b/docs/concepts/buckconfig.md @@ -224,3 +224,52 @@ exists. [cxx#other_platform] cxxppflags="-D MYMACRO=\"Watchman\"" ``` + +## Sections + +Below is an incomplete list of supported buckconfigs. + +## [alias] + +This section contains definitions of [build target](build_target.md) aliases. + +``` +[alias]app = //apps/myapp:app + apptest = //apps/myapp:test +``` + +These aliases can then be used from the command line: + +``` +$ buck2 build app +$ buck2 test apptest +``` + +## [repositories] + +Lists the cells that constitute the Buck2 project. Buck2 builds that are part of +this project—that is, which use this `.buckconfig`—can access the cells +specified in this section. + +``` +[repositories] + buck = . + bazel_skylib = ./third-party/skylark/bazel-skylib +``` + +The string on the left-hand side of the equals sign is the _alias_ for the cell. +The string on the right-hand side of the equals sign is the path to the cell +from the directory that contains this `.buckconfig` file. It is not necessary to +include the current cell in this section, but we consider it a best practice to +do so: + +``` +buck = . +``` + +You can view the contents of this section using the `buck2 audit cell` command. +Although the name of the section is _repositories_, the section actually lists +_cells_. In practice, Buck cells often correspond to repositories, but this is +not a requirement. For more information about the relationship between Buck +projects, cells, and repositories, see the [Key Concepts](key_concepts.md) +topic. diff --git a/docs/concepts/build_file.md b/docs/concepts/build_file.md new file mode 100644 index 0000000000000..8a4279bdde20f --- /dev/null +++ b/docs/concepts/build_file.md @@ -0,0 +1,63 @@ +--- +id: build_file +title: Build File +--- + +# Build File + +A _build file_ is a file, typically named `BUCK`, that defines one or more +[build rule](build_rule.md)s. Note that you can change the name that Buck2 uses +for the build file in the `buildfile` section of `.buckconfig`. A source file in +your project can only be referenced by rules in its "nearest" build file, where +"nearest" means its closest direct ancestor in your project's file tree. (If a +source file has a build file as a sibling, then that is its nearest ancestor.) +For example, if your project had the following `BUCK` files: + +``` +java/com/facebook/base/BUCK +java/com/facebook/common/BUCK +java/com/facebook/common/collect/BUCK +``` + +Then your build rules would have the following constraints: + +- Rules in `java/com/facebook/base/BUCK` can reference any file under + `java/com/facebook/base/`. +- Rules in `java/com/facebook/common/` can reference any files under that + directory, except for those under `java/com/facebook/common/collect/`, as + those "belong" to the `BUCK` file in the `collect` directory. + +The set of source files accessible to a build file is also known as its _build +package_. The way to refer to code across build packages is to create build +rules and use `deps` to refer to that code. Going back to the previous example, +suppose code in `java/com/facebook/common/concurrent/` wants to depend on code +in `java/com/facebook/common/collect/`. Presumably +`java/com/facebook/common/collect/BUCK` has a build rule like: + +``` +java_library( + name = 'collect', + srcs = glob(['*.java']), + deps = ['//java/com/facebook/base:base',],) +``` + +Then `java/com/facebook/common/BUCK` could have a rule like: + +``` +java_library( + name = 'concurrent', + srcs = glob(['concurrent/*.java']), + deps = ['//java/com/facebook/base:base','//java/com/facebook/common/collect:collect',],) +``` + +whereas the following **would be invalid** because +`java/com/facebook/common/collect/` has its own build file, so +`//java/com/facebook/common/collect:concurrent` cannot list +`java/com/facebook/common/collect/*.java` in its `srcs`. + +``` +java_library( + name = 'concurrent', + srcs = glob(['collect/*.java', 'concurrent/*.java']), + deps = ['//java/com/facebook/base:base',],) +``` diff --git a/docs/concepts/build_rule.md b/docs/concepts/build_rule.md new file mode 100644 index 0000000000000..aeda0a1eaa6e5 --- /dev/null +++ b/docs/concepts/build_rule.md @@ -0,0 +1,126 @@ +--- +id: build_rule +title: Build Rule +--- + +# Build Rule + +A _build rule_ is a procedure for producing output files from a set of input +files in the context of a specified build configuration. Build rules are +specified in [build file](build_file.md)s—typically named BUCK. **Note:** A +build rule must explicitly specify, in its arguments, all of its required inputs +in order for Buck2 to be able to build the rule's output in a way that is +deterministic and reproducible. + +## Buck2's collection of build rules + +Buck2 comes with a collection of built-in build rules for many common build +procedures. For example, compiling Java code against the Android SDK is a common +procedure, so Buck2 provides the build rule +[`android_library`](../../api/rules/#android_library) to do that. Similarly, the +final product of most Android development is an APK, so you can use the build +rule [`android_binary`](../../api/rules/#android_binary) to create an APK. + +## Source files as inputs to build rules + +Most build rules specify source files as inputs. For example, a +[`cxx_library`](../../api/rules/#cxx_library) rule would specify `.cpp` files as +inputs. To support specifying these files, a `cxx_library` rule provides the +`srcs` argument. Some languages, such as C++, use header files as well. To +specify these, `cxx_library` provides a `headers` argument. In addition to +`srcs` and `headers`, some rules provide variants of these arguments, such as +`platform_srcs` and `platform_headers`. These arguments support groups of source +files that should be used as inputs only when building for specific platforms. + +### Package boundaries and access to source files + +In Buck2, a BUCK file defines a _package_, which corresponds _roughly_ to the +directory that contains the BUCK file and those subdirectories that do not +themselves contain BUCK files. (To learn more, see the +[Key Concepts](key_concepts.md) topic.) A rule in a BUCK file cannot specify a +source file as an input unless that source file is in that BUCK file's package. +An exception to this restriction exists for header files, but only if a rule in +the package that contains the header file _exports_ that header file using the +`exported_headers` argument. For more details, see the description for +`exported_headers` in, for example, the +[`cxx_library`](../../api/rules/#cxx_library) topic. More commonly though, the +package for a BUCK file contains all the source files required for the rules +defined in that BUCK file. Functionality in source files from other packages is +made available through the artifacts produced by the rules in the BUCK files for +those packages. For example, a [`cxx_binary`](../../api/rules/#cxx_binary) might +use the functionality in a `cxx_library` that is defined in another package. To +access that functionality, the `cxx_binary` would take that `cxx_library` as a +_dependency_. + +##### Symlinks: Use with caution if at all + +We recommend that you do _not_ use symlinks—either absolute or relative—to +specify input files to build rules. Although using symlinks in this context does +sometimes work, it can lead to unexpected behavior and errors. + +## Dependencies: Output from one rule as input to another rule + +A build rule can use the output from another build rule as one of its inputs by +specifying that rule as a _dependency_. Typically, a build rule specifies its +dependencies as a list of [build target](build_target.md)s in its `deps` +argument. However, the rule can also specify dependencies—as build targets—in +other arguments, such as `srcs`. **Example:** The output of a +[`java_library`](../../api/rules/#java_library) rule is a JAR file. If a +`java_library` rule specifies another `java_library` rule as a dependency, the +JAR file produced by the specified rule is added to the classpath for the +`java_library` that depends on it. **Example:** If a +[`java_binary`](../../api/rules/#java_binary) rule specifies a `java_library` +rule as a dependency, the JAR file for the specified `java_library` is available +on the classpath for the `java_binary`. In addition, in the case of +`java_binary`, the JAR files for any dependencies of the `java_library` rule +_are also_ made available to the `java_binary` rule—and if those dependencies +have dependencies of their own, they are added as well. This exhaustive cascade +of dependencies is referred to as the rule's _transitive closure_. + +### Required dependencies are always built first + +Buck2 guarantees that any dependencies that a rule lists that are required in +order to build that rule are built successfully _before_ Buck2 builds the rule +itself. Note though that there can be special cases—such as +[`apple_bundle`](../../api/rules/#apple_bundle)—where a rule's listed +dependencies do not actually need to be built before the rule. + +### Visibility + +In order for a build rule to take a dependency on another build rule, the build +rule on which the dependency is taken must be _visible_ to the build rule taking +the dependency. A build rule's `visibility` argument is a list of +[build target pattern](target_pattern.md)s that specify the rules that can take +that rule as a dependency. For more information about the concept of visibility +in Buck2, see the [Visibility](visibility.md) topic. + +### Dependencies define a graph + +Build rules and their dependencies define a directed acyclic graph (DAG). Buck2 +requires this graph to be acyclic to make it possible to build independent +subgraphs in parallel. + +## How to handle special cases: genrules and macros + +Although Buck2 provides a rich set of built-in build rules for developers, it is +not able to address all possible needs. As an "escape hatch," Buck2 provides a +category of generic build rules called _genrules_. With genrules, you can +perform arbitrary operations using shell scripts. The genrules supported by +Buck2 are: + +- [`genrule`](../../api/rules/#genrule) +- [`apk_genrule`](../../api/rules/#apk_genrule) +- [`cxx_genrule`](../../api/rules/#cxx_genrule) + +### Multiple output files with genrules + +In most cases, a build rule produces exactly one output file. However, with +genrules, you can specify an output _directory_ and write arbitrary files to +that directory. + +### Macros + +Finally, note that you can define functions that generate build rules. In +general, this should not be something that you need to do, but taking advantage +of this option might help you add needed functionality to Buck2's without +editing its source code. diff --git a/docs/concepts/build_target.md b/docs/concepts/build_target.md new file mode 100644 index 0000000000000..fccef48d9a778 --- /dev/null +++ b/docs/concepts/build_target.md @@ -0,0 +1,129 @@ +--- +id: build_target +title: Build Target +--- + +# Build Target + +A _build target_ is a string that identifies a build rule in your project. Build +targets are used as arguments to Buck2 commands, such as +[`buck2 build`](../../users/commands/build) and +[`buck2 run`](../../users/commands/run). Build targets are also used as +arguments to [build rules](build_rule.md) to enable one build rule to reference +another. For example, a build rule might use a build target to reference another +rule in order to specify that rule as a _dependency_. + +#### Fully-qualified build targets + +Here is an example of a _fully-qualified_ build target: + +``` +//java/com/facebook/share:ui +``` + +A fully-qualified build target has three components: + +1. The `//` prefix indicates that the subsequent path is from the _root_ of your + project. You can use the `buck2 root` command to identify the root of your + project. +2. The `java/com/facebook/share` between the `//` prefix and the colon (`:`) + indicates that the [build file](build_file.md) (usually named `BUCK`) is + located in the directory `java/com/facebook/share`. +3. The `ui` after the colon (`:`) indicates the name of the build rule within + the build file. Build rule names must be unique within a build file. By + _name_ we mean, more formally, the value of the `name` argument to the build + rule. + +Note that the name of the build file itself—usually BUCK—does _not_ occur in the +build target. All build files within a given Buck2 project must have the same +name—defined in the `[buildfile].name` entry of `.buckconfig`. Therefore, it is +unnecessary to include the name in the target. The full regular expression for a +fully-qualified build target is as follows: + +``` +[A-Za-z0-9._-]*//[A-Za-z0-9/._-]*:[A-Za-z0-9_/.=,@~+-]+ +|- cell name -| | package path | |--- rule name ----| +``` + +In Buck2, a _cell_ defines a directory tree of one or more Buck2 packages. For +more information about Buck2 cells and their relationship to packages and +projects, see the [Key Concepts](key_concepts.md) topic. **NOTE:** All target +paths are assumed to start from the root of the Buck2 project. Buck2 does not +support specifying a target path that starts from a directory below the root. +Although the double forward slash (`//`) that prefixes target paths can be +omitted when specifying a target from the command line (see **Pro Tips** below), +Buck2 still assumes that the path is from the root. Buck2 does support +_relative_ build paths, but in Buck2, that concept refers to specifying build +targets _from within_ a build file. See **Relative build targets** below for +more details. + +#### Relative build targets + +A _relative_ build target can be used to reference a [build rule](build_rule.md) +_within the same _[_build file_](build_file.md). A relative build target starts +with a colon (`:`) and is followed by only the third component (or _short name_) +of the fully-qualified build target. The following snippet from a build file +shows an example of using a relative path. + +``` +## Assume this rule is in //java/com/facebook/share/BUCK# +java_binary( + name = 'ui_jar', + deps = [## The following target path## //java/com/facebook/share:ui## is the same as using the following relative path.#':ui',],) +``` + +## Command-line Pro Tips + +Here are some ways that you can reduce your typing when you specify build +targets as command-line arguments to the `buck2 build` or `buck2 run` commands. +Consider the following example of a fully-qualified build target used with the +`buck2 build` command: + +``` +buck2 build //java/com/facebook/share:share +``` + +Although Buck2 is always strict when parsing build targets in build files, Buck2 +is flexible when parsing build targets on the command-line. Specifically, the +leading `//` is optional on the command line, so the above could be: + +``` +buck2 build java/com/facebook/share:share +``` + +Also, if there is a forward slash before the colon, it is ignored, so this could +also be written as: + +``` +buck2 build java/com/facebook/share/:share +``` + +which enables you to produce the red text shown below using tab-completion, +which dramatically reduces how much you need to type: + +``` +buck2 build java/com/facebook/share/:share +``` + +Finally, if the final path element matches the value specified after the colon, +it can be omitted: + +``` +# This is treated as //java/com/facebook/share:share. +buck2 build java/com/facebook/share/ +``` + +which makes the build target even easier to tab-complete. For this reason, the +name of the build rule for the primary deliverable in a build file is often +named the same as the parent directory. That way, it can be built from the +command-line with less typing. + +## See also + +Buck2 supports the ability to define **_aliases_ for build targets**; using +aliases can improve brevity when specifying targets on the Buck2 command line. +For more information, see the [`[alias]`](buckconfig.md#alias) section in the +documentation for [`.buckconfig`](buckconfig.md). A +[**build target pattern**](target_pattern.md) is a string that describes a set +of one or more build targets. For example, the pattern `//...` is used to build +an entire project. For more information, see the **Build Target Pattern** topic. diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md index 688103772edd5..fd654f0e83925 100644 --- a/docs/concepts/glossary.md +++ b/docs/concepts/glossary.md @@ -160,10 +160,7 @@ invocations are executed from the project root. Data returned from a [rule](#rule) function. It's the only way that information from this rule is available to other rules that depend on it (see -[dependency](#dependency)). Every rule must return at least the `DefaultInfo` -provider. A common case is to also return either `RunInfo` (because they are -executable) or custom providers that the dependents rule can use. For more -information, see +[dependency](#dependency)). For more information, see [Providers](https://buck2.build/docs/rule_authors/writing_rules/#providers). #### Remote execution (RE) @@ -206,6 +203,12 @@ article. The Buck2 project maintains and uses an open source [Starlark interpreter in Rust](https://github.com/facebookexperimental/starlark-rust). +#### Subtarget + +Collection of [providers](#provider) that can be accesed by name. The subtargets +can have their own subtargets as well, which can be accessed by chaining them, +e.g.: `buck2 build cell//foo:bar[baz][qux]`. + #### Target An object that is defined in a [BUCK file](#buck-file). Targets represent the diff --git a/docs/concepts/key_concepts.md b/docs/concepts/key_concepts.md new file mode 100644 index 0000000000000..36d87285e48be --- /dev/null +++ b/docs/concepts/key_concepts.md @@ -0,0 +1,85 @@ +--- +id: key_concepts +title: Key Concepts +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; + +# Key concepts + +Buck2 has a number of fundamental concepts: + +- A [**_build rule_**](build_rule.md) describes how to produce an output file + from a set of input files. Most build rules are specific to a particular + language or platform. For example, you would use the + [`cxx_binary`](../../api/rules/#cxx_binary) rule to create a C++ binary, but + you would use the [`android_binary`](../../api/rules/#android_binary) rule to + create an Android APK. +- A [**_build target_**](build_target.md) is a string that uniquely identifies a + build rule. It can be thought of as a URI for the build rule within the Buck2 + project. +- A [**_build file_**](build_rule.md) defines one or more build rules. In Buck2, + build files are typically named `BUCK`. A `BUCK` file is analogous to the + `Makefile` used by the Make utility. In your project, you will usually have a + separate `BUCK` file for each buildable unit of software—such as a binary or + library. For large projects, you could have hundreds of `BUCK` files. + +A Buck2 **_package_** comprises of: a Buck2 build file (a `BUCK` file), all +files—such as source files and headers—in the same directory as the `BUCK` file +or in subdirectories, provided those subdirectories do not themselves contain a +`BUCK` file. To say it another way, a `BUCK` file defines the root of a package, +but Buck2 packages might not include all their subdirectories because Buck2 +packages do not overlap or contain other Buck2 packages. For example, in the +following diagram, the BUCK file in directory `app-dir-1` defines that directory +as the root of a package—which is labeled **Package A** in the diagram. The +directory `app-dir-2` is part of Package A because it is a subdirectory of +`app-dir-1`, but does not itself contain a BUCK file. Now, consider directory +`app-dir-3`. Because `app-dir-3` contains a BUCK file it is the root of a new +package (**Package B**). Although `app-dir-3` is a subdirectory of `app-dir-1`, +it is _not_ part of Package A. Buck2 has the concept of a **_cell_**, which +defines a directory tree of one or more Buck2 packages. A Buck2 build could +involve multiple cells. Cells often correspond to repositories, but this isn't +required. The root of a Buck2 cell contains a global configuration file called +[**`.buckconfig`**](buckconfig.md). Note that although the cell root should +contain a `.buckconfig`, the presence of a `.buckconfig` file doesn't in itself +define a cell. Rather, _the cells involved in a build are defined at the time +Buck2 is invoked_; they are specified in the `.buckconfig` for the Buck2 +_project_ (see below). A Buck2 **_project_** is defined by the `.buckconfig` +where Buck2 is invoked, or if that directory doesn't contain a `.buckconfig`, +the project is defined by the `.buckconfig` in the nearest ancestor directory. +The `.buckconfig` for the project specifies the cells that constitute the Buck2 +project. Specifically, these cells are specified in the +`[repositories]`(buckconfig.md#repositories) section of the `.buckconfig`. Note +that the directory tree rooted at this `.buckconfig` is automatically considered +a cell by Buck2; in other words, the project's `.buckconfig` doesn't need to +specify the project cell explicitly—although it is a good practice to do so. + +justifyContent + +### Buck2's dependency graph + +Every build rule can have zero or more dependencies. You can specify these +dependencies using, for example, the `deps` argument to the build rule. For more +information about specifying dependencies, consult the reference page for the +build rule you are using. These dependencies form a directed graph, called the +_target graph_. Buck2 requires the graph to be acyclic. When building the output +of a build rule, all of the rule's transitive dependencies are built first. This +means that the graph is built in a "bottom-up" fashion. A build rule knows only +which rules it depends on, not which rules depend on it. This makes the graph +easier to reason about and enables Buck2 to identify independent subgraphs that +can be built in parallel. It also enables Buck2 to determine the minimal set of +build targets that need to be rebuilt. + +### Multiple Buck2 projects in a single repository + +Buck2 is designed to build multiple deliverables from a single repository—that +is, a _monorepo_—rather than from multiple repositories. Support for the +monorepo design motivated Buck2's support for cells and projects. It is +Facebook's experience that maintaining all dependencies in the same repository +makes it easier to ensure that all developers have the correct version of the +code and simplifies the process of making atomic commits. + +### See also + +Take a look at the [Concept Map](concept_map.md) for a visualization of how +Buck2 concepts interact with each other. Also see the [Glossary](glossary.md). diff --git a/docs/concepts/target_pattern.md b/docs/concepts/target_pattern.md index e067bd9926cb4..59fa79665b2cb 100644 --- a/docs/concepts/target_pattern.md +++ b/docs/concepts/target_pattern.md @@ -6,7 +6,7 @@ title: Target Pattern A _target pattern_ is a string that resolves to a set of [targets](./glossary.md#target). A target pattern can be used as arguments to commands, such as `buck2 build` and `buck uquery`. You can also use build target -patterns in the [visibility](./glossary.md#visibility)) argument of your build +patterns in the [visibility](./glossary.md#visibility) argument of your build [rules](./glossary.md#rule). The simplest build target pattern matches the build target of the same name: diff --git a/docs/developers/bxl_basics.md b/docs/developers/bxl_basics.md index 15c8975d382cb..ffe0358b6d084 100644 --- a/docs/developers/bxl_basics.md +++ b/docs/developers/bxl_basics.md @@ -10,6 +10,70 @@ in BXL may be challenging without much prior knowledge of Buck2 building blocks ## Common BXL functionalities +### Build + +You can build targets within BXL with +[`ctx.build()`](../../api/bxl/bxl_ctx/#bxl_ctxbuild). The result is a +[`bxl_build_result`](../../api/bxl/bxl_build_result), which has `artifacts()` +and `failures()` functions that provide iterators to the artifacts or failures, +respectively. You can pass in a single target or target pattern to build. + +### Analysis + +You can run analysis on targets within BXL via +[`ctx.analysis()`](../../api/bxl/bxl_ctx/#bxl_ctxanalysis). Analysis means to +evaluate the underlying rule implementation for the inputted targets, and +produce the providers that the rule defined for the target. A common workflow is +to inspect the resulting providers, and perhaps ensure parts of these providers +or run actions using information from the providers (see [Actions](#actions) +below). + +### Query + +Buck2 supports a couple different query types: querying the unconfigured graph +(`buck2 uquery`), the configured graph (`buck2 cquery`), or the action graph +(`buck2 aquery`). These queries are all available in BXL as well: + +- `ctx.uquery()` returns a [`uqueryctx`](../../api/bxl/uqueryctx) +- `ctx.cquery()` returns a [`cqueryctx`](../../api/bxl/cqueryctx) +- `ctx.aquery()` returns a [`aqueryctx`](../../api/bxl/aqueryctx) + +You can read more about the individual queries in the API docs. There are many +queries that are common between uquery, cquery, and aquery, but cquery and +aquery will have extra queries unique to the configured graph or the action +graph. One more thing to call out is the `eval()` query, which is a special +query that takes in the entire query as a string literal. A common use for +`eval()` is to migrate a complex query from Buck2 CLI to BXL by dropping the +entire query string directly into `eval()`. + +The query results are target sets (iterable container) of +[`unconfigured_target_node`s](../../api/bxl/unconfigured_target_node) for +uquery, [`target_node`s](../../api/bxl/target_node) for cquery, and +[`action_query_node`s](../../api/bxl/action_query_node) for aquery. Each of +these node types have accessors on their attributes. A common workflow is to run +some query in BXL, and iterate through the resulting nodes to inspect their +attributes, and use those attributes to inform further computations in BXL. + +#### Uquery + +Querying the unconfigured graph means that no configurations (such as platforms +and transitions) have been applied to the target graph yet. This means that it's +very possible that some parts of the target graph is broken due to lack of +configurations. Generally to avoid this problem, cquery may be preferred +instead. + +#### Cquery + +Querying the configured graph means that configurations have been applied to the +target graph. For cquery, we require that users use a +[target universe](../developers/bxl_target_universe.md) for their query inputs. + +#### Aquery + +Aquery is a quite different from uquery and cquery. It is used to query the +action graph, which is constructed after Buck2 runs analysis on the targets and +produces the list of providers and actions needed to build the target. + ### Actions You can create actions directly within the BXL API. The available action APIs diff --git a/docs/developers/bxl_common_how_tos.md b/docs/developers/bxl_common_how_tos.md index 11984f98ccdcd..4bc69ce1be544 100644 --- a/docs/developers/bxl_common_how_tos.md +++ b/docs/developers/bxl_common_how_tos.md @@ -210,3 +210,25 @@ mocking. (`print()` and `ctx.output.print()`). - **Test** - the main method to test a BXL script is to actually invoke it with required inputs then verify the outputs. + +## Getting the path of an artifact as a string + +The starlark `artifact` type encapsulates source artifacts, declared artifacts, +and build artifacts. It can be dangerous to access paths and use them in further +BXL computations. For example, if you are trying to use absolute paths for +something and end up passing it into a remotely executed action, the absolute +path may not exist on the remote machine. Or, if you are working with paths and +expecting the artifact to already have been materialized in further BXL +computations, that would also result in errors. + +However, if you are not making any assumptions about the existence of these +artifacts, you can use use +[`get_path_without_materialization()`](../../api/bxl/globals#get_path_without_materialization), +which accepts source, declared, or build aritfacts. It does _not_ accept ensured +artifacts (also see +[What do I need to know about ensured artifacts](./bxl_faq.md#what-do-i-need-to-know-about-ensured-artifacts)). + +For getting paths of `cmd_args()` inputs, you can use +[`get_paths_without_materialization()`](../../api/bxl/globals#get_paths_without_materialization), +but note this is risky because the inputs could contain tsets, which, when +expanded, could be very large. Use these methods at your own risk. diff --git a/docs/developers/bxl_target_universe.md b/docs/developers/bxl_target_universe.md index dfd8e7004b655..c13c5d9cc24ea 100644 --- a/docs/developers/bxl_target_universe.md +++ b/docs/developers/bxl_target_universe.md @@ -102,6 +102,19 @@ def _impl: rdeps = ctx.cquery().rdeps(universe_node, from_node) ``` +### `keep-going` + +The configured graph can be broken for various reasons: incompatible targets +(BXL skips these automatically), visibility issues, nonexistent targets, etc. +For issues that are not incompatible targets, the `target_universe` can be +constructed with the `keep_going` flag set to `True` to skip any other errors, +and your cquery will not error out. Note that `keep_going` is only compatible +for a single string literal target or target pattern at the moment. + +```python +ctx.target_universe("//foo/...", keep_going = True) +``` + ## BXL build and target universe Note that BXL builds currently do not support target universe, but we intend to diff --git a/docs/getting_started.md b/docs/getting_started.md index 993b8d0e432d1..f269eb20b96a7 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -12,8 +12,8 @@ To get started, first install [rustup](https://rustup.rs/), then compile the `buck2` executable: ```bash -rustup install nightly-2023-10-01 -cargo +nightly-2023-10-01 install --git https://github.com/facebook/buck2.git buck2 +rustup install nightly-2023-11-10 +cargo +nightly-2023-11-10 install --git https://github.com/facebook/buck2.git buck2 ``` The above commands install `buck2` into a suitable directory, such as diff --git a/docs/index.md b/docs/index.md index 3db9021ff74a0..9b44429717164 100644 --- a/docs/index.md +++ b/docs/index.md @@ -93,6 +93,13 @@ your team. ### External videos about Buck2 +- [Accelerating builds with Buck2](https://www.youtube.com/watch?v=oMIzKVxUNAE) + Neil talks about why Buck2 is fast. +- [Buck2: optimizations & dynamic dependencies](https://www.youtube.com/watch?v=EQfVu42KwDs) + Neil and Chris talk about why Buck2 is fast and some of the advanced + dependency features. +- [Building Erlang with Buck2](https://www.youtube.com/watch?v=4ALgsBqNBhQ) + Andreas talks about building WhatsApp with Buck2. - [antlir2: Deterministic image bulids with Buck2](https://www.youtube.com/watch?v=Wv-ilbckSx4) talks about layering a packaging system over Buck2. @@ -123,56 +130,4 @@ your project. - [Notes for Developers](developers/developers.fb.md) - more advanced workflows and notes around debugging, profiling etc. -## Specialised groups - -We have Workplace groups and task tags for various projects. Most task folders -are _not monitored_, so post all questions and bug reports to a Workplace group. - -### Workplace groups - -- [Admarket](https://fb.workplace.com/groups/2011248092366093) - collaboration - between Admarket, DevX and Build Infra teams in their effort to migrate - Admarket to Buck2. -- [Android](https://fb.workplace.com/groups/4318511658259181) - discussions on - anything related to the migration of fbandroid to Buck2. -- [Apple](https://fb.workplace.com/groups/305599448025888/) - discussions - related to the migration of fbobjc to Buck2. -- [Fbcode TD](https://fb.workplace.com/groups/603286664133355/) - migrations for - TDs, including fbcode, mobile, and rl TDs, as well as UTD. -- [Fbcode](https://fb.workplace.com/groups/1080276222750085) - collaboration - between fbcode teams, DevX and Build Infra in their effort to migrate fbcode - services to Buck2. -- [Hack](https://fb.workplace.com/groups/496546384752884) - discussions, ideas, - updates, and more as we move Hack to Buck2. -- [Haskell](https://fb.workplace.com/groups/202582585277200/) - discussions, - ideas, updates, and more as we move Haskell to Buck2. -- [Infer](https://fb.workplace.com/groups/601798364244831/) - discussions - related to ideas, bugs, jobs, and feedback on Infer. -- [Open source](https://fb.workplace.com/groups/3434452653448246) - people - particularly enthusiastic about open sourcing Buck2. -- [Reality labs](https://fb.workplace.com/groups/930797200910874/) - unmoderated - non-support group for talking about arvr's integration and onboarding to - Buck2. -- [Shots](https://fb.workplace.com/groups/4899204743424118) - Shots engineers - who are experimenting with Buck2. -- [Tpx](https://fb.workplace.com/groups/900436963938958/) - Buck2/Tpx - coordination group. -- [Unicorn](https://fb.workplace.com/groups/503973410692177) - collaboration - between Unicorn, DevX and Build Infra teams in their effort to migrate Unicorn - to Buck2. -- [WhatsApp](https://fb.workplace.com/groups/whatsapp.buck2) - Buck2 in the - WhatsApp server. -- [Windows](https://fb.workplace.com/groups/580747310463852/) - discussions - related to Buck2 on Windows. - -### Task folders - -- [Admarket on Buck V2](https://www.internalfb.com/tasks?q=163089765955500) -- [Apple Build Infra](https://www.internalfb.com/tasks?q=1710478139132259) -- [Buck2](https://www.internalfb.com/tasks?q=446583836738538) -- [DICE - BuckV2](https://www.internalfb.com/tasks?q=413466250534831) -- [Eden on Buck V2](https://www.internalfb.com/tasks?q=406698320868619) -- [FbCode TD on Buck2](https://www.internalfb.com/tasks?q=980682532796984) -- [Unicorn on Buck V2](https://www.internalfb.com/tasks?q=262220628906648) - diff --git a/docs/rfcs/audit_visibility.md b/docs/rfcs/audit_visibility.md index 86621b6a57bb2..deeaf033e0f29 100644 --- a/docs/rfcs/audit_visibility.md +++ b/docs/rfcs/audit_visibility.md @@ -2,12 +2,12 @@ ## Context -Buck has a concept of Visibility for every -target. It allows users to define, for each target, the targets it can depend on -and targets that can depend on it. Visibility is specified as an allowlist of -targets/target patterns, and any target used that falls outside of the allowlist -fails visibility checking. Visibility pattern can be specified on `visibility` -and `within_view` attributes in buildfiles and +Buck has a concept of Visibility for every target. It allows users to define, +for each target, the targets it can depend on and targets that can depend on it. +Visibility is specified as an allowlist of targets/target patterns, and any +target used that falls outside of the allowlist fails visibility checking. +Visibility pattern can be specified on `visibility` and `within_view` attributes +in buildfiles and [PACKAGE files](https://www.internalfb.com/intern/wiki/Buck-users/Key_Concepts/Package_Files/). Visibility is important to lots of codebase maintainers because it can be used diff --git a/docs/rfcs/drafts/cfg-modifiers/api.md b/docs/rfcs/drafts/cfg-modifiers/api.md index e0a23ad9b5c71..afc4fc0d2cf58 100644 --- a/docs/rfcs/drafts/cfg-modifiers/api.md +++ b/docs/rfcs/drafts/cfg-modifiers/api.md @@ -132,26 +132,26 @@ constraint value for that setting. For example, specifying `cfg//os:linux` as a modifier will insert `cfg//os:linux` into the configuration, overriding any existing constraint value for the `cfg//os:_` constraint setting. -Another type of modifier is a `modifier_select()` of a constraint value. This -can change the constraint value inserted based on the existing configuration. -For example, a modifier like +Another type of modifier is the `modifiers.match()` operator. This operator can +change the constraint value inserted based on the existing configuration. For +example, a modifier like ```python -modifier_select({ +modifiers.match({ "cfg//os:windows": "cfg//compiler:msvc", "DEFAULT": "cfg//compiler:clang", }) ``` will insert msvc constraint into the configuration if OS is windows or clang -constraint otherwise. A `modifier_select()` behaves similarly to Buck's -`select()` but can only be used in a modifier. A `modifier_select()` can only be -used to modify a single constraint setting, so the following example is not -valid. +constraint otherwise. A `modifiers.match()` behaves similarly to Buck's +`select()` but can only be used in a modifier context. A `modifiers.match()` can +only be used to modify a single constraint setting, so the following example is +not valid. ```python # This fails because a modifier cannot modify both compiler and OS. -modifier_select({ +modifiers.match({ "cfg//os:windows": "cfg//compiler:msvc", "DEFAULT": "cfg//os:linux", }) @@ -179,7 +179,7 @@ targets in repo. set_cfg_modifiers(modifiers = [ "cfg//:linux", - modifier_select({ + modifiers.match({ "DEFAULT": "cfg//compiler:clang", "cfg//os:windows": "cfg//compiler:msvc", }), @@ -209,7 +209,7 @@ Then the same PACKAGE modifiers can be specified as follows. set_cfg_modifiers(modifiers = [ "linux", - modifier_select({ + modifiers.match({ "DEFAULT": "clang", "windows": "msvc", }), @@ -292,7 +292,7 @@ Suppose modifiers for `repo//foo:bar` are specified as follows. set_cfg_modifiers(modifiers = [ "cfg//:linux", - modifier_select({ + modifiers.match({ "DEFAULT": "cfg//compiler:clang", "cfg//os:windows": "cfg//compiler:msvc", }), @@ -319,7 +319,7 @@ For OS, the linux modifier from `repo/PACKAGE` will apply first, followed by macos modifier from `repo/foo/PACKAGE` and windows modifier from `repo//foo:bar`'s target modifiers, so `repo//foo:bar` will end up with `cfg//os:windows` for `cfg//os:_` in its configuration. Next, to resolve -compiler modifier, the `modifier_select` from `repo/PACKAGE` will resolve to +compiler modifier, the `modifiers.match` from `repo/PACKAGE` will resolve to `cfg//compiler:msvc` since existing configuration is windows and apply that as the modifier. The target configuration for `repo//foo:bar` ends up with windows and msvc. @@ -327,7 +327,7 @@ and msvc. However, suppose user invokes `repo//foo:bar?linux` on the command line. When resolving OS modifier, the linux modifier from cli will override any existing OS constraint and insert linux into the configuraiton. Then, when resolving the -compiler modifier, the `modifier_select` will resolve to `cfg//compiler:clang`, +compiler modifier, the `modifiers.match` will resolve to `cfg//compiler:clang`, giving clang and linux as the final configuration. Because command line modifiers will apply at the end, they are also known as @@ -335,35 +335,36 @@ required modifiers. Any modifier specified on the command line will always override any modifier for the same constraint setting specified in the repo. The ordering of constraint setting to resolve modifiers is determined based on -dependency order of constraints specified in the keys of the `modifier_select` -specified. Because some modifiers select on other constraints, modifiers for +dependency order of constraints specified in the keys of the `modifiers.match` +specified. Because some modifiers match on other constraints, modifiers for those constraints must be resolved first. In the previous example, because -compiler modifier selects on OS constraints, Buck will resolve all OS modifiers -before resolving compiler modifiers. `modifier_select` that ends up with a cycle -of selected constraints (ex. compiler modifier selects on sanitizer but -sanitizer modifier also selects on compiler) will be an error. +compiler modifier matches on OS constraints, Buck will resolve all OS modifiers +before resolving compiler modifiers. `modifiers.match` that ends up with a cycle +of matched constraints (ex. compiler modifier matches on sanitizer but sanitizer +modifier also matches on compiler) will be an error. -### Modifier-Specific Selects +### Modifier Matches -Modifiers have 3 types of select operators that allow for powerful compositions. -Each operation is a function that accepts a dictionary where the keys are -conditionals and values are modifiers. +Modifiers have 3 types of `match` operators that allow for powerful +compositions. Each operation is a function that accepts a dictionary where the +keys are conditionals and values are modifiers. -1. `modifier_select`. Introduced in the previous sections, this is capable of +1. `modifiers.match`. Introduced in the previous sections, this is capable of inserting constraints based on constraints in the existing configuration. -2. `rule_select`. This is capable of selecting based on the rule name (also - known as rule type). The keys are regex patterns to match against the rule - name or "DEFAULT". Partial matches are allowed. +2. `modifiers.match_rule`. This is capable of selecting based on the rule name + (also known as rule type). The keys are regex patterns to match against the + rule name or "DEFAULT". Partial matches are allowed. -3. `host_select`. This selects based on the host configuration, whereas - `modifier_select` selects based on the target configuration. This host - configuration is constructed when resolving modifiers. `host_select` is - important to making `buck build` work anywhere on any platform. For example, - when the OS to configure is not specified, it's best to assume that the user - wants to target the same OS as the host machine. +3. `modifiers.match_host`. This selects based on the host configuration, whereas + `modifiers.match` selects based on the target configuration. This host + configuration is constructed when resolving modifiers. `modifiers.match_host` + is important to making `buck build` work anywhere on any platform. For + example, when the OS to configure is not specified, it's best to assume that + the user wants to target the same OS as the host machine. -An example using `rule_select` and `host_select` is as follows. +An example using `modifiers.match_rule` and `modifiers.match_host` is as +follows. ```python # root/PACKAGE @@ -376,7 +377,7 @@ An example using `rule_select` and `host_select` is as follows. # configuration. set_cfg_modifiers(modifiers = [ - rule_select({ + modifiers.match_rule({ "apple_.*": "cfg//os:iphone", "android_.*": "cfg//os:android", "DEFAULT": host_select({ diff --git a/docs/rfcs/drafts/universal-cfg-naming.md b/docs/rfcs/drafts/universal-cfg-naming.md new file mode 100644 index 0000000000000..8295bf2189d5b --- /dev/null +++ b/docs/rfcs/drafts/universal-cfg-naming.md @@ -0,0 +1,61 @@ +# Universal Configuration Naming Function + +_tl;dr:_ This RFC proposes using a single naming function to generate names for +all configurations. + +## Context + +NOTE: The configuration name consists of a readable string followed by the hash +of the configuration. The readable string is technically the `PlatformInfo` +name. For sake of ease of writing, this doc uses configuration name and platform +name interchangeably to describe this concept. + +Currently, there are 3 ways to create and name a configuration. + +1. A `platform` target defines a configuration, and the platform target label + becomes the platform name. +2. A transition function defines the configuration and generates a name for the + configuration. +3. When a modifier is used, the cfg constructor function for modifiers defines + the configuration and its name. There is currently a single naming function + that generates all modifier-based configuration names. + +Modifiers are intended to replace platforms, so in the future all configuration +names will be generated. Unfortuately, most of the generated names today used +today in transitions are not very good. Problems that I've seen in practice +include: + +1. Configuration names barely contain any useful information about the + configuration. This happens a lot in transitions. For example, the android + split CPU architecture transition names the generated configurations "x86_64" + and "arm64", which tells very little about the configuration beyond the CPU + architectures it splits on. +2. Transition function incorrectly retains the old configuration name that is no + longer relevant, misleading the user about what this configuration actually + does. I've seen this happen where a configuration has py3.8 in name but the + python version constraint stored is actually py3.10. + +## Proposal + +Register a single Starlark function to define all configuration names. This +Starlark function would accept a `ConfigurationInfo` and return a string for the +name of the `ConfigurationInfo`. + +```python +# Example +def name(cfg: ConfigurationInfo) -> str: + # ... +``` + +`PlatformInfo` is no longer available in Starlark. Any place that previously +uses a `PlatformInfo` will now use `ConfigurationInfo` instead. Buck2 will +invoke this function each time it encounters a new `ConfigurationInfo` to define +its name. + +This function will attempt to provide a useful name based on the constraints in +the configuration, which mitigates the issue of short or misleading +configuration names. There are some risks that there will be high amount of code +complexity in a function if all configurations are named by one function. + +This function will most likely be registered via a `set_cfg_name` function or +something callable from root PACKAGE file or potentially prelude. diff --git a/docs/rule_authors/alias.md b/docs/rule_authors/alias.md index e12128f110821..4be02fbed3ca1 100644 --- a/docs/rule_authors/alias.md +++ b/docs/rule_authors/alias.md @@ -34,7 +34,6 @@ alias( The `versioned_alias` rule has the following relevant attributes: - `name` - (required) what the `actual`'s label should be aliased as. -- `actual` - (required) a target label. - `versions` - (required) a map of versions to their respective versioned target labels. diff --git a/docs/users/build_observability/build_report.md b/docs/users/build_observability/build_report.md index cee163e17b13c..570183d81e643 100644 --- a/docs/users/build_observability/build_report.md +++ b/docs/users/build_observability/build_report.md @@ -92,9 +92,6 @@ ConfiguredBuildReportEntry { } Error { - # TO BE DEPRECATED - message: str, - # The stringified hash of the same stringified error message that is shown to the user on the # console. The hash is stored as the key in the `strings` cache of the `BuildReport` message_content: str, diff --git a/docs/users/cheatsheet.md b/docs/users/cheatsheet.md new file mode 100644 index 0000000000000..8ddf61889f2a4 --- /dev/null +++ b/docs/users/cheatsheet.md @@ -0,0 +1,130 @@ +--- +id: cheat_sheet +title: Cheat Sheet +--- + +# Buck2 Cheat Sheet + +This section provides example command lines that you can use to obtain +information about Buck2 and about your build. These techniques can help you to +understand how your build works and to troubleshoot issues with your build. +These examples use the [`buck2 cquery`](../query/cquery) command. We recommend +cquery over uquery in most cases because cquery operates on the configured +graph, which means that targets have had the expected configurations applied on +them. + +--- + +- How do I see the arguments for a given rule from the command line? +- How do I find all the targets for a package? +- How do I specify more than one target to `buck2 cquery`? +- How do I get the attribute names and values for the targets that result from a + query? +- How do I perform a query inside of a rule? +- How do I find the dependencies for a target, that is, the targets on which a + specified target depends? +- How do I find the reverse-dependencies for a target, that is, the targets that + depend on a specified target? +- How do I find the build file that contains the target that owns a source file? + +--- + +### How do I find all the targets for a package? + +Specify a _build target pattern_ that represents the targets in the package. + +``` +buck2 cquery //path/to/dir/... +``` + +The `buck2 cquery` command can accept a +[build target pattern](../../concepts/target_pattern) as a parameter. If you +specify a build target pattern, Buck2 evaluates this pattern and shows all the +build targets that match it. + +### How do I specify more than one target to `buck2 cquery`? + +Use the `buck2 cquery set()` operator. The following command line returns the +target `main` in the build file in the root of the Buck2 project and all the +targets from the build file in the `myclass` subdirectory of the root. + +``` +buck2 cquery "set( '//:main' '//myclass:' )" +``` + +### How do I get the attribute names and values for the targets returned by a query? + +Add the `--output-attribute ` or `--output-all-attributes` option to +the command line, followed by regular expressions that represent the attributes +of interest. + +``` +buck2 cquery "deps(//foo:bar)" --output-attribute 'name' 'exported_headers' +``` + +The `--output-attribute` option enables you to specify which attributes Buck2 +should return. Instead of returning the names of the targets that match the +query expression, Buck2 returns the names and values of the specified attributes +for those targets in JSON format. Attributes are specified as regular +expressions. For example, `'.*'` matches all attributes. See the +[`buck2 cquery` docs](../query/cquery) for more details. The output for the +example query above might look something like the following. + +``` +{"//foo/bar/lib:lib" : {"exported_headers" : [ "App/util.h" ],"name" : "lib"},"//foo/bar:app" : {"exported_headers" : [ "App/lib.h" ],"name" : "app"}} +``` + +### How do I perform a query** \***inside**\* **of a rule? + +Buck2 supports certain string parameter macros to be used when defining a +target. You can use the query macros as such: + +``` +$(query_targets "queryfunction(//:foo)") +$(query_outputs "queryfunction(//:foo)") +$(query_targets_and_outputs [SEPARATOR] "queryfunction(//:foo)") +``` + +Note, however, that the query macros are supported only for +[`genrule`](../../api/rules/#genrule) and +[`apk_genrule`](../../api/rules/#apk_genrule). + +### How do I find the dependencies for a target? + +Use the `deps()` operator. + +``` +buck2 cquery "deps('//foo:bar')" +buck2 cquery "deps('//foo:bar', 1, first_order_deps())" +buck2 cquery "deps(set('//foo:bar' '//foo:lib' '//foo/baz:util'))" +``` + +The `deps` operator finds the dependencies of the specified targets. The first +argument represents the targets of interest. This can be a single +[build target](../../concepts/build_target) or +[build target pattern](../../concepts/target_pattern), or a set of these. The +optional second argument is the _depth_ of the search for dependencies from the +specified targets. For example, `1`, as shown in the example above, returns only +the direct dependencies. If you do not provide this argument, the output is the +complete set of transitive dependencies. How do I find the reverse-dependencies +for a target, that is, the targets that** \***depend on**\* **a specified +target? Use the `buck2 cquery rdeps()` (reverse dependencies) operator. The +following example, returns the targets in the +[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) of +`//foo:bar` that depend directly on `//example:baz`. + +``` +buck2 cquery "rdeps('//foo:bar', '//example:baz', 1)" +``` + +### How do I find the buildfile that contains the target that owns a source file? + +In order to find the build file associated with a source file, combine the +`owner` operator with `buildfile`. For example, + +``` +buck2 cquery "buildfile(owner('foo/bar/main.cpp'))" +``` + +first finds the targets that _own_ `foo/bar/main.cpp` and then returns the build +files, such as `foo/bar/BUCK`, that define those targets. diff --git a/docs/users/faq/buck_hanging.md b/docs/users/faq/buck_hanging.md new file mode 100644 index 0000000000000..660558c6bca40 --- /dev/null +++ b/docs/users/faq/buck_hanging.md @@ -0,0 +1,83 @@ +--- +id: buck_hanging +title: Why is Buck2 hanging? +--- + +Let's look at how to troubleshoot when buck2 hangs, i.e. it just sits there +saying "Jobs: In progress: 0, ..." but it’s not finishing... + +When buck2 hangs, there are two possibilities: It’s either hanging doing +_something_, or it’s hanging doing _nothing_. The first thing you should do is +figure out which of those is happening. That’s because the tools to debug either +of those are _very_ different! We will mainly focus on the first in this case. + +To figure out which hang you have on your hands, just look at how much CPU buck2 +is using when the hang occurs using your favorite activity monitor (e.g. `top`, +`htop`). Remember that you can find the buck2 daemon’s PID using `buck2 status`. +Ideally, break the utilization down by threads (in top, that’s `top -Hp $PID`). + +If any thread is using 100% CPU for some period of time, then you probably have +a busy hang (buck2 is doing “something”) which are usually easier to debug. + +## How to debug a “busy” hang + +### Getting a stack trace + +When debugging a busy hang, the first thing to do is to work out what the +process is doing. There are many tools you can use for this (like a profiler), +but the absolute simplest one is quickstack: just run `quickstack -p $PID`, and +it’ll show you a stack dump for all the threads in your process. If you prefer +`gdb`, you can use `gdb -p $PID`, then `thread apply all bt`, and that’s the +same thing. + +Note that a stack trace tells you what the process is doing at a point in time, +so don’t just look at the very last frame and call it the culprit. Instead, look +at the stack as a whole. If you need more perspective, use a sampling profiler +(strobeclient run --pid $PID). You can also +just grab stack traces at a few points in time and see if they look similar: +this is exactly what a sampling profiler does, albeit at a higher frequency. + +### Interpreting the stack trace + +Let's consider an example user report ( see +[here](https://fb.workplace.com/groups/buck2users/permalink/3232782826978076/)) +with the following stack trace: + +``` +#01 0x0000000005b1ec26 in as core::iter::traits::iterator::Iterator>::next () from ... +#02 0x0000000005b23998 in as itertools::Itertools>::exactly_one () from ... +#03 0x00000000059dbb2c in buck2_server_commands::commands::build::create_unhashed_outputs () from ... +#04 0x0000000005c3c677 in ::command::{closure#0}> as core::future::future::Future>::poll () from ... +#05 0x00000000054c58a3 in as buck2_server_ctx::ctx::ServerCommandDiceContext>::with_dice_ctx::{closure#0}::{closure#0}::{closure#0}, core::pin::Pin> + core::marker::Send>>, cli_proto::BuildResponse>::{closure#0}> as core::future::future::Future>::poll () from ... +#06 0x00000000054c7ae3 in ::{closure#0}::{closure#0}> as core::future::future::Future>::poll () from ... +#07 0x0000000005370df8 in ::call_in_span::, buck2_data::CommandEnd)>, ::span_async::{closure#0}::{closure#0}>, core::result::Result>::{closure#0}::{closure#0}::{closure#0}> () from ... +#08 0x00000000054f7288 in ::build::{closure#0}> as core::future::future::Future>::poll () from... + ... +``` + +At this point, you can look at the code, and note that there is no span around +the output symlink creation function (`create_unhashed_outputs`). This suggests +you’ve found your culprit: there is indeed a buck2 bug and we’re spending ages +creating unhashed output symlinks, and since you need a span to get any console +feedback, the console says nothing is happening. + +**An easy fix**: In this particular instance, Thomas spotted +[an easy optimization](https://github.com/facebook/buck2/commit/d677e41253b73a31aafa1255a532c38992482efd) +which resolved the issue. But, of course, that’s not always possible. If the +easy fix hadn't been available, we’d be at a dead end, so what do we do next? + +**A harder fix**: If it is not clear what the root-cause is, you can +bisect[you can bisect](users/faq/how_to_bisect.fb.md): +i.e. do a binary search across commits for the commit that introduced a given +breakage/perf degradation. Thanks to the fact that we enforce a +linear history, bisecting is pretty straightforward in +`fbsource`. Then, once you identify their commit that caused +breakage, investigate what caused the issue. + +## How to debug a “doing nothing” hang + +**Cycle in dependencies**: If buck2 seems to be doing nothing (e.g. CPU usage is +0%), one of the reasons could be a cycle in your dependencies, which may cause +buck2 to hang (buck2 does implement a form of cycle detection, but it +unfortunately has false negatives). You can confirm this by running buck1, which +will report cycles properly. diff --git a/docs/users/faq/starlark_peak_mem.md b/docs/users/faq/starlark_peak_mem.md new file mode 100644 index 0000000000000..920a520d7e4c4 --- /dev/null +++ b/docs/users/faq/starlark_peak_mem.md @@ -0,0 +1,168 @@ +--- +id: starlark_peak_mem +title: Debugging Excess Starlark Peak Memory +--- + +## Wut memory? + +Peak memory is the maximum amount of memory used during evaluation of that +particular Starlark file. The memory is usually released after we finish the +evaluation of the file. Because Starlark is only garbage collected in between +top-level statements in the BUCK file, but not garbage collected inside function +calls/macros, on large servers with 64 hardware threads (or more), memory usage +might accumulate, causing slowdowns or OOMs or even SEVs (e.g. +S372092). See +[this post](https://fb.workplace.com/groups/1267349253953900/permalink/1312921066063385/) +for more details on how Starlark's current GC works . + +To prevent such issues until proper GC is implemented, we have set a hard `2GB` +memory limit for Starlark's evaluation of build files. This is a per-file limit. + +Note that this is different than the actual process memory which might include +other things apart from Starlark’s evaluation. + +## How do I see my build file's peak memory usage? + +To see the Starlark peak memory usage of a build file, you can inspect the event +log for your build file. Here is an example entry from the event log for buck2 +uquery `target` showing that it uses 1.5GB: + +``` +{"Event":{..."data":{"Load":{"module_id":"target:BUCK","cell":"...","error":null,"starlark_peak_allocated_bytes":1610608640}}}}}} +``` + +## Profiler to the rescue! + +If you want to see more detailed breakdown where the memory is used, you should +profile Starlark's evaluation of build files. See +[this page](../../rule_authors/optimization.md/#starlark-profiling) for details +of profiling in the loading stage. This is a great starting point for +troubleshooting. + +## How do I reduce memory footprint? + +There are many reasons why Starlark's evaluation of your build file might use a +lot of memory. We list a few common cases below but there might be more +cases. See +[this post](https://fb.workplace.com/groups/buck2eng/permalink/3309329642697846/) +for a few real world examples of debugging Starlark peak memory usage of core +Android macros that have saved over 5.7GB peak memory! + +High level guidance is to pay attention to loops as a starting point. Are there +any unnecessary computations? Can you shave them off? + +### Repeatedly allocating memory unnecessarily in a loop + +A common case where memory usage might accumulate is repeatedly allocating +memory in a loop. For instance, below we call a memory intensive function in a +loop unnecessarily: + +``` +for target in huge_target_list: + memory_intensive_fun(x,y) + ... +``` + +Instead, if we know that arguments `x` and `y` don't change, we could hoist the +call to `memory_intensive_fun` outside of the loop as follows: + +``` +memory_intensive_fun(x,y) +for target in huge_target_list: + ... +``` + +### Simply allocating very big data-structures! + +Another reason why Starlark uses a lot of memory could simply be bacause the +build file allocates a very big-data structure. For instance, below we allocate +a list with 1 billion integers! + +``` +million_list = [1 for i in range(1 << 20)] +billion_list = million_list * (1 << 10) + +``` + +As a workaround, could you think of splitting the list? + +### Algorithmically inefficient code + +Another reason could be because memory efficiency of your code is bad, i.e. you +are unnecessarily allocating a lot of memory. Let's look at an example where we +try to process a bunch of targets inefficiently: + +``` +targets = generate_targets(n) +for target in targets: + process(target) + +``` + +If `targets` list is big **and** each target takes a lot of space in memory, +memory usage might exceed the limit. Instead, a more efficient version might be +to process each target as you generate it: + +``` +for i in range(n): + target = generate_target(i) + process(target) +``` + +In this version, each target is processed as it is generated so we never need to +store more than one target in memory. + +### Usage of inefficient library calls + +A more subtle reason could be unknowingly invoking library calls that allocate +each time they are called. A well-known case of this is the `dict.items()` call. + +``` +for project, version in constraints.items(): + # process each project .... +``` + +We do an allocation on every call to `constraints.items()`. Especially if this +is a hot code in Starlark, this could cause an OOM. Instead, a potential fix is +to hoist the call out: + +``` +constraints = constraints.items() +for project, version in constraints: + # process each project .... +``` + +However, you need to ensure that the dictionary is not mutated inside, otherwise +you would get functionally different code. A similar case occurs for +`dict.keys()` where it returns a new list for containing the keys. + +### Allocating for rare cases + +Finally, another pattern is allocating memory for the rare cases. For instance, +consdier the following example + +``` +for target in huge_target_list: + if memory_intensive_condition([target]) + fail(...) +``` + +Above program could be optimized as follows: + +``` +if memory_intensive_condition(huge_target_list) + for target in huge_target_list: + if memory_intensive_condition([target]) + fail(...) +``` + +so that in the common non-failure case, we don't end up allocating excessive +memory. + +## I still need more help! + +If you still can not figure out how to reduce Starlark memory footprint of your +build files, please post in +[Buck2 Users](https://fb.workplace.com/groups/buck2users)raise +[an issue](https://github.com/facebook/buck2/issues) in our Github +project. diff --git a/examples/README.md b/examples/README.md index e0c6614d5f5b3..79bbce77e3c27 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,23 +1,22 @@ # buck2 examples -In these folders are some examples on how to get buck2 working with -your favorite languages and tools. +In these folders are some examples on how to get buck2 working with your +favorite languages and tools. ## with_prelude -Examples taking advantage of the prelude to create toolchain-independent -build definitions in cpp and python. Includes as an example a usecase -for building and using c-extension-backed python libraries. +Examples taking advantage of the prelude to create toolchain-independent build +definitions in cpp and python. Includes as an example a usecase for building and +using c-extension-backed python libraries. -Note: to take advantage of these examples you must symlink the prelude -into this folder. +Note: to take advantage of these examples you must symlink the prelude into this +folder. ## no_prelude -Preludeless examples for those wanting to use buck2 with their own -rules and toolchains. In here you can learn about how BUILD -files interact with rules, and how the provider abstraction can be -used to encapsulate build logic. +Preludeless examples for those wanting to use buck2 with their own rules and +toolchains. In here you can learn about how BUILD files interact with rules, and +how the provider abstraction can be used to encapsulate build logic. ## toolchains diff --git a/examples/bootstrap/README.md b/examples/bootstrap/README.md index aa60fe87849db..8dbb02ae09dca 100644 --- a/examples/bootstrap/README.md +++ b/examples/bootstrap/README.md @@ -1,11 +1,14 @@ # Configuring a bootstrapping toolchain setup -This project provides an example of what it might look like to configure a bootstrapping toolchain and construct a different toolchain using an artifact built with the former. +This project provides an example of what it might look like to configure a +bootstrapping toolchain and construct a different toolchain using an artifact +built with the former. ## How to build 1. Build or install `buck2` with Cargo -2. This project assumes Rust, Clang, and Python to be present. See `toolchains/BUCK` for how we pull those in from the system. +2. This project assumes Rust, Clang, and Python to be present. See + `toolchains/BUCK` for how we pull those in from the system. 3. Run `buck2 init --git` 4. Run commands: e.g. `buck2 run :hello_world`, `buck2 build //...` @@ -13,7 +16,9 @@ This project provides an example of what it might look like to configure a boots ### Bootstrap constraint -In order to differentiate between a regular toolchain and a bootstrap toolchain, we introduce a new constraint setting `bootstrap//:bootstrap` and a corresponding constraint value `bootstrap//:use_bootstrap`. +In order to differentiate between a regular toolchain and a bootstrap toolchain, +we introduce a new constraint setting `bootstrap//:bootstrap` and a +corresponding constraint value `bootstrap//:use_bootstrap`. ```python constraint_setting( @@ -28,7 +33,9 @@ constraint_value( ### Bootstrap platform -We then define a new platform `bootstrap//platform:bootstrap`, which inherits everything from the default platform `bootstrap//platform:default` and adds the extra `bootstrap` constraint defined above. +We then define a new platform `bootstrap//platform:bootstrap`, which inherits +everything from the default platform `bootstrap//platform:default` and adds the +extra `bootstrap` constraint defined above. ```python platform( @@ -40,12 +47,15 @@ platform( ### Bootstrap toolchain -We are using Rust for this example, but the concept is not specific to Rust. Our goal is to -build a Rust compiler with the bootstrap toolchain, construct a new toolchain with the compiler, -then build a Rust binary with the newly built Rust compiler. For simplicity, we are not building -an actual Rust compiler, but using a small wrapper Rust binary that execs into the compiler picked from the system. +We are using Rust for this example, but the concept is not specific to Rust. Our +goal is to build a Rust compiler with the bootstrap toolchain, construct a new +toolchain with the compiler, then build a Rust binary with the newly built Rust +compiler. For simplicity, we are not building an actual Rust compiler, but using +a small wrapper Rust binary that execs into the compiler picked from the system. + +First, we setup a bootstrap toolchain using the `system_rust_toolchain` provided +in the prelude. -First, we setup a bootstrap toolchain using the `system_rust_toolchain` provided in the prelude. ```python system_rust_toolchain( name = "rust_bootstrap_toolchain", @@ -53,6 +63,7 @@ system_rust_toolchain( ``` Then, we configure a build for our "rustc". + ```python rust_binary( name = "rustc_wrapper", @@ -60,7 +71,9 @@ rust_binary( ) ``` -To construct a new toolchain that uses the new "rustc", we use `configured_alias` to tack on the `bootstrap` to the binary. +To construct a new toolchain that uses the new "rustc", we use +`configured_alias` to tack on the `bootstrap` to the binary. + ```python rust_toolchain( name = "rust_toolchain_with_compiled_rustc", @@ -74,7 +87,9 @@ configured_alias( ) ``` -Now that we have both toolchains constructed, we can create our final Rust toolchain that switches between the two based on the `use_bootstrap` constraint. +Now that we have both toolchains constructed, we can create our final Rust +toolchain that switches between the two based on the `use_bootstrap` constraint. + ```python toolchain_alias( name = "rust", diff --git a/examples/hello_world/README.md b/examples/hello_world/README.md index 2f346e313642f..7ca7d66cbdb7d 100644 --- a/examples/hello_world/README.md +++ b/examples/hello_world/README.md @@ -1,10 +1,14 @@ ## A simple Hello World project using the buck2-prelude -This example demonstrates how a simple C++ project might be built with Buck2 using the prelude. +This example demonstrates how a simple C++ project might be built with Buck2 +using the prelude. + +In the `toolchains` cell, we define two toolchains needed: +`system_cxx_toolchain` and `system_python_bootstrap_toolchain`, both pulled in +from the prelude. The `BUCK` file at the project root contain a `cxx_binary` +target and its `cxx_library` dependency. `.buckconfig` contains the +configuration to set the target platform for the project: -In the `toolchains` cell, we define two toolchains needed: `system_cxx_toolchain` and `system_python_bootstrap_toolchain`, both pulled in from the prelude. -The `BUCK` file at the project root contain a `cxx_binary` target and its `cxx_library` dependency. -`.buckconfig` contains the configuration to set the target platform for the project: ``` [parser] target_platform_detector_spec = target:root//...->prelude//platforms:default diff --git a/examples/no_prelude/README.md b/examples/no_prelude/README.md index 21dbf878ceb4b..6f4f1d3ea32cc 100644 --- a/examples/no_prelude/README.md +++ b/examples/no_prelude/README.md @@ -1,5 +1,9 @@ ## No-prelude example -This is an example project that does not rely on https://github.com/facebook/buck2-prelude. Instead the prelude cell points to a `prelude` directory with an empty `prelude.bzl` file, like so: + +This is an example project that does not rely on +https://github.com/facebook/buck2-prelude. Instead the prelude cell points to a +`prelude` directory with an empty `prelude.bzl` file, like so: + ``` #.buckconfig [repositories] @@ -7,10 +11,13 @@ root = . prelude = prelude ``` -All rules and toolchains are defined manually within each of the subdirectories. (e.g. `cpp/rules.bzl`, `cpp/toolchain.bzl`) +All rules and toolchains are defined manually within each of the subdirectories. +(e.g. `cpp/rules.bzl`, `cpp/toolchain.bzl`) ## Sample commands + Install Buck2, cd into a project, and run + ```bash # List all targets buck2 targets //... diff --git a/examples/no_prelude/toolchains/go_toolchain.bzl b/examples/no_prelude/toolchains/go_toolchain.bzl index 5a0ea76ae1ed9..94a7719d1ef30 100644 --- a/examples/no_prelude/toolchains/go_toolchain.bzl +++ b/examples/no_prelude/toolchains/go_toolchain.bzl @@ -57,10 +57,16 @@ def _download_toolchain(ctx: AnalysisContext): script_content.append(cmd_args(output, format = "mkdir {}")) else: script_content.append(cmd_args(output, format = "mkdir -p {}")) - script_content.extend([ - cmd_args(output, format = "cd {}"), - cmd_args(["tar", compress_flag, "-x", "-f", archive], delimiter = " ").relative_to(output), - ]) + if host_info().os.is_windows: + script_content.extend([ + cmd_args(output, format = "cd {}"), + cmd_args(["unzip", archive], delimiter = " ").relative_to(output), + ]) + else: + script_content.extend([ + cmd_args(output, format = "cd {}"), + cmd_args(["tar", compress_flag, "-x", "-f", archive], delimiter = " ").relative_to(output), + ]) script, _ = ctx.actions.write( script_name, script_content, @@ -80,6 +86,7 @@ def _download_toolchain(ctx: AnalysisContext): def _toolchain_config(): version = "1.20.7" os = host_info().os + arch = host_info().arch if os.is_windows: return struct( sha256 = "736dc6c7fcab1c96b682c8c93e38d7e371e62a17d34cb2c37d451a1147f66af9", @@ -88,12 +95,22 @@ def _toolchain_config(): version = version, ) if os.is_macos: - return struct( - sha256 = "eea1e7e4c2f75c72629050e6a6c7c46c446d64056732a7787fb3ba16ace1982e", - platform = "darwin-arm64", - archive_extension = "tar.gz", - version = version, - ) + if arch.is_aarch64: + return struct( + sha256 = "eea1e7e4c2f75c72629050e6a6c7c46c446d64056732a7787fb3ba16ace1982e", + platform = "darwin-arm64", + archive_extension = "tar.gz", + version = version, + ) + elif arch.is_x86_64: + return struct( + sha256 = "785170eab380a8985d53896808b0a71336d0ea60e0a26099b4ccec77798b1cf4", + platform = "darwin-amd64", + archive_extension = "tar.gz", + version = version, + ) + else: + fail("unrecognized architecture: couldn't select macOS go toolchain") # Default linux return struct( diff --git a/examples/remote_execution/buildbarn/README.md b/examples/remote_execution/buildbarn/README.md index 2dbe380234618..4a4f8103dd1e0 100644 --- a/examples/remote_execution/buildbarn/README.md +++ b/examples/remote_execution/buildbarn/README.md @@ -1,10 +1,12 @@ ## Remote execution integration with Buildbarn -This project provides a small example of what a project that utilizes [Buildbarn](https://github.com/buildbarn). +This project provides a small example of what a project that utilizes +[Buildbarn](https://github.com/buildbarn). -In this document, we will go over the key configs used in this setup. -Using a local docker-compose deployment from the [example deployment repo](https://github.com/buildbarn/bb-deployments). -If you already have a Buildbarn deployment you can skip that. +In this document, we will go over the key configs used in this setup. Using a +local docker-compose deployment from the +[example deployment repo](https://github.com/buildbarn/bb-deployments). If you +already have a Buildbarn deployment you can skip that. ### Deploy a local Buildbarn @@ -15,14 +17,14 @@ If you already have a Buildbarn deployment you can skip that. .../bb-deployments/docker-compose $ ./run.sh ``` -This uses `docker-compose` to spin up the required Buildbarn services. -Using FUSE based workers, those are generally the fastest as they can load action files on demand -and avoids the overhead of setting up the full input root up front. -In practice many actions do not read all the files in the input root. +This uses `docker-compose` to spin up the required Buildbarn services. Using +FUSE based workers, those are generally the fastest as they can load action +files on demand and avoids the overhead of setting up the full input root up +front. In practice many actions do not read all the files in the input root. If you do not want FUSE workers you can instead switch to hardlinking workers -The example deployments have two worker kinds "fuse", and "hardlinking", -you can see the queues in the Buildbarn scheduler, http://localhost:7982. +The example deployments have two worker kinds "fuse", and "hardlinking", you can +see the queues in the Buildbarn scheduler, http://localhost:7982. ``` Buildbarn Scheduler @@ -38,7 +40,8 @@ Instance name Platform properties ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448" ``` -More information is available in the [repo](https://github.com/buildbarn/bb-deployments). +More information is available in the +[repo](https://github.com/buildbarn/bb-deployments). ### Relevant configs in .buckconfig @@ -59,5 +62,5 @@ TLS is not used in this example. ### Relevant configs in `ExecutionPlatformInfo` -Buildbarn takes in a Docker image and `OSFamily` in its RE properties to select a worker. -This is configured in `root//platforms:platforms`. +Buildbarn takes in a Docker image and `OSFamily` in its RE properties to select +a worker. This is configured in `root//platforms:platforms`. diff --git a/examples/remote_execution/buildbuddy/README.md b/examples/remote_execution/buildbuddy/README.md index 011a9398b3ed1..f50c9cbb5cc8e 100644 --- a/examples/remote_execution/buildbuddy/README.md +++ b/examples/remote_execution/buildbuddy/README.md @@ -1,12 +1,14 @@ ## Remote execution integration with BuildBuddy -This project provides a small example of what a project that utilizies [BuildBuddy](https://www.buildbuddy.io/)'s RE might look like. +This project provides a small example of what a project that utilizies +[BuildBuddy](https://www.buildbuddy.io/)'s RE might look like. In this document, we will go over the key configs used in this setup. ### Relevant configs in .buckconfig -First, the BuildBuddy endpoint and api key should be configured as the following: +First, the BuildBuddy endpoint and api key should be configured as the +following: ```ini [buck2_re_client] @@ -18,5 +20,7 @@ http_headers = x-buildbuddy-api-key:$BUILDBUDDY_API_KEY ### Relevant configs in `ExecutionPlatformInfo` -BuildBuddy takes in a Docker image and OSFamily in its execution platform's execution properties(`exec_properties`) to select an executor. -The execution platform used in this project `root//platforms:platforms` uses the `container-image` key to set this up. +BuildBuddy takes in a Docker image and OSFamily in its execution platform's +execution properties(`exec_properties`) to select an executor. The execution +platform used in this project `root//platforms:platforms` uses the +`container-image` key to set this up. diff --git a/examples/remote_execution/engflow/README.md b/examples/remote_execution/engflow/README.md index c84964edfbd94..383f629c0121a 100644 --- a/examples/remote_execution/engflow/README.md +++ b/examples/remote_execution/engflow/README.md @@ -1,12 +1,14 @@ ## Remote execution integration with EngFlow -This project provides a small example of what a project that utilizes [EngFlow](https://www.engflow.com/)'s RE offering might look like. +This project provides a small example of what a project that utilizes +[EngFlow](https://www.engflow.com/)'s RE offering might look like. In this document, we will go over the key configs used in this setup. ### Relevant configs in .buckconfig -First, the EngFlow endpoint and certificate should be configured as the following: +First, the EngFlow endpoint and certificate should be configured as the +following: ```ini [buck2_re_client] @@ -17,6 +19,7 @@ tls_client_cert = $ENGFLOW_CERTIFICATE ``` Additionally, set the `digest_algorithm` config to `SHA256`. + ```ini [buck2] digest_algorithms = SHA256 @@ -24,5 +27,6 @@ digest_algorithms = SHA256 ### Relevant configs in `ExecutionPlatformInfo` -EngFlow takes in a Docker image as its execution platform. -The execution platform used in this project `root//platforms:platforms` uses the `container-image` key to set this up. +EngFlow takes in a Docker image as its execution platform. The execution +platform used in this project `root//platforms:platforms` uses the +`container-image` key to set this up. diff --git a/examples/toolchains/cxx_zig_toolchain/toolchains/README.md b/examples/toolchains/cxx_zig_toolchain/toolchains/README.md index 990a4e6ff7952..0280fabf27649 100644 --- a/examples/toolchains/cxx_zig_toolchain/toolchains/README.md +++ b/examples/toolchains/cxx_zig_toolchain/toolchains/README.md @@ -1,15 +1,17 @@ This example tests the `zig cc` based self-contained C/C++ toolchain. To build it within the open source tree of buck2 to you need to -* Create a symlink for the prelude + +- Create a symlink for the prelude ``` ln -s ../../../prelude prelude ``` -* Remove the top-level `.buckconfig` +- Remove the top-level `.buckconfig` ``` rm ../../../.buckconfig ``` -* Apply the following patch to the prelude +- Apply the following patch to the prelude + ``` diff --git a/prelude/cxx/tools/TARGETS.v2 b/prelude/cxx/tools/TARGETS.v2 index 2030d2f..5db1689 100644 diff --git a/examples/with_prelude/README.md b/examples/with_prelude/README.md index 301c59ee65d9c..801800359ee67 100644 --- a/examples/with_prelude/README.md +++ b/examples/with_prelude/README.md @@ -17,7 +17,11 @@ Now all targets aside from OCaml related ones are ready to be built. The information in this section is (at this time) Linux and macOS specific. -The commands in `ocaml-setup.sh` assume an activated [opam](https://opam.ocaml.org/) installation. Their effect is to create a symlink in the 'third-party/opam' directory. This symlink supports building the example OCaml targets. If the symlink is found to already exist, it will not be overwritten. +The commands in `ocaml-setup.sh` assume an activated +[opam](https://opam.ocaml.org/) installation. Their effect is to create a +symlink in the 'third-party/opam' directory. This symlink supports building the +example OCaml targets. If the symlink is found to already exist, it will not be +overwritten. ## Sample commands diff --git a/examples/with_prelude/haskell/BUCK b/examples/with_prelude/haskell/BUCK index 0f44458d8edcd..91cc5a5ed47cf 100644 --- a/examples/with_prelude/haskell/BUCK +++ b/examples/with_prelude/haskell/BUCK @@ -1,12 +1,14 @@ +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect haskell_library( name = "library", srcs = ["Library.hs"], -) if host_info().os.is_linux else None +) if _SUPPORTED else None # buildifier: disable=no-effect haskell_binary( name = "main", srcs = ["Main.hs"], deps = [":library"], -) if host_info().os.is_linux else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/calc/BUCK b/examples/with_prelude/ocaml/calc/BUCK index 1641dd9e40710..8b3a310c261c8 100644 --- a/examples/with_prelude/ocaml/calc/BUCK +++ b/examples/with_prelude/ocaml/calc/BUCK @@ -1,3 +1,5 @@ +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect ocaml_binary( name = "calc", @@ -6,4 +8,4 @@ ocaml_binary( "lexer.mll", "parser.mly", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/embed/BUCK b/examples/with_prelude/ocaml/embed/BUCK index 113e594918ed9..cb2262fa2d493 100644 --- a/examples/with_prelude/ocaml/embed/BUCK +++ b/examples/with_prelude/ocaml/embed/BUCK @@ -1,10 +1,12 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect ocaml_object( name = "fib-ml", srcs = ["fib.ml"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect cxx_binary( @@ -14,7 +16,7 @@ cxx_binary( ":fib-ml", "//third-party/ocaml:ocaml-dev", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect rust_binary( @@ -23,18 +25,18 @@ rust_binary( crate_root = "fib.rs", link_style = "static", deps = [":fib-ml"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "check-fib-cpp", command = "$(exe_target :fib-cpp)", output = "fib(10) = Result is: 89", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "check-fib-rs", command = "$(exe_target :fib-rs)", output = "fib(10) = Result is: 89", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/extend/BUCK b/examples/with_prelude/ocaml/extend/BUCK index 99f2e1e0cce6b..c13643d3fbe69 100644 --- a/examples/with_prelude/ocaml/extend/BUCK +++ b/examples/with_prelude/ocaml/extend/BUCK @@ -1,5 +1,7 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect ocaml_binary( name = "hello-c", @@ -7,7 +9,7 @@ ocaml_binary( "hello.ml", ], deps = [":hello-stubs-c"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect cxx_library( @@ -16,7 +18,7 @@ cxx_library( "hello_stubs.c", ], deps = ["//third-party/ocaml:ocaml-dev"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_binary( @@ -25,7 +27,7 @@ ocaml_binary( "hello.ml", ], deps = [":hello-stubs-rs"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect rust_library( @@ -34,18 +36,18 @@ rust_library( "hello_stubs.rs", ], crate_root = "hello_stubs.rs", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "check-hello-c", command = "$(exe_target :hello-c)", output = "Hello C", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "check-hello-rs", command = "$(exe_target :hello-rs)", output = "Hello Rust", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/hello_world/BUCK b/examples/with_prelude/ocaml/hello_world/BUCK index 63fd74496fc94..46e2e71801226 100644 --- a/examples/with_prelude/ocaml/hello_world/BUCK +++ b/examples/with_prelude/ocaml/hello_world/BUCK @@ -1,22 +1,24 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect ocaml_binary( name = "hello-world", srcs = ["hello_world.ml"], deps = [":hello-world-lib"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_library( name = "hello-world-lib", srcs = ["hello_world_lib.ml"], visibility = ["PUBLIC"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "hello-world-check", command = "$(exe_target :hello-world)", output = "Hello world!", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/ppx/BUCK b/examples/with_prelude/ocaml/ppx/BUCK index 000b749610e4c..078e3bc0f3989 100644 --- a/examples/with_prelude/ocaml/ppx/BUCK +++ b/examples/with_prelude/ocaml/ppx/BUCK @@ -1,5 +1,7 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect ocaml_library( name = "ppx-record-selectors", @@ -7,7 +9,7 @@ ocaml_library( deps = [ "//third-party/ocaml:ppxlib", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_binary( @@ -15,11 +17,15 @@ ocaml_binary( srcs = ["ppx_driver.ml"], compiler_flags = [ "-linkall", + "-cclib", + "-L/opt/homebrew/lib", + "-cclib", + "-lzstd", ], deps = [ ":ppx-record-selectors", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_binary( @@ -29,7 +35,7 @@ ocaml_binary( "-ppx", "$(exe_target :ppx) --as-ppx", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # [Note: Use the 'expand' sub-target to see the effects of # preprocessor expansion] @@ -53,4 +59,4 @@ assert_output( name = "ppx-record-selectors-test-check", command = "$(exe_target :ppx-record-selectors-test)", output = "4 quux", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/wrap/BUCK b/examples/with_prelude/ocaml/wrap/BUCK index 33eb0626429f8..b04902388d874 100644 --- a/examples/with_prelude/ocaml/wrap/BUCK +++ b/examples/with_prelude/ocaml/wrap/BUCK @@ -1,5 +1,7 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + export_file( name = "mylib.mli", src = "mylib.mli", @@ -22,7 +24,7 @@ ocaml_library( "-49", ], visibility = [":mylib"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_library( @@ -46,7 +48,7 @@ ocaml_library( ], visibility = ["PUBLIC"], deps = [":mylib__"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_binary( @@ -54,11 +56,11 @@ ocaml_binary( srcs = ["test_Mylib.ml"], visibility = ["PUBLIC"], deps = [":mylib"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "test-Mylib-check", command = "$(exe_target :test-Mylib)", output = "Hello world!", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/examples/with_prelude/ocaml/wrap/with-masking/BUCK b/examples/with_prelude/ocaml/wrap/with-masking/BUCK index f9f71aa9e3899..ef252822d453b 100644 --- a/examples/with_prelude/ocaml/wrap/with-masking/BUCK +++ b/examples/with_prelude/ocaml/wrap/with-masking/BUCK @@ -1,5 +1,7 @@ load("//test_utils.bzl", "assert_output") +_SUPPORTED = not host_info().os.is_windows + # buildifier: disable=no-effect export_file( name = "al__.mli", @@ -7,7 +9,7 @@ export_file( ":al__", ":al__imp", ], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_library( @@ -18,7 +20,7 @@ ocaml_library( ], compiler_flags = ["-no-alias-deps"], visibility = [":al__imp"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_library( @@ -39,7 +41,7 @@ ocaml_library( ], visibility = [":al"], deps = [":al__"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_library( @@ -51,7 +53,7 @@ ocaml_library( ], visibility = ["PUBLIC"], deps = [":al__imp"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect ocaml_binary( @@ -59,11 +61,11 @@ ocaml_binary( srcs = ["test_Al.ml"], visibility = ["PUBLIC"], deps = [":al"], -) if not host_info().os.is_windows else None +) if _SUPPORTED else None # buildifier: disable=no-effect assert_output( name = "test-Al-check", command = "$(exe_target :test-Al)", output = "Hello world!", -) if not host_info().os.is_windows else None +) if _SUPPORTED else None diff --git a/gazebo/README.md b/gazebo/README.md index a41cf04d5b0b4..23da12907b59d 100644 --- a/gazebo/README.md +++ b/gazebo/README.md @@ -6,41 +6,75 @@ [![docs.rs availability](https://img.shields.io/docsrs/gazebo?label=docs.rs)](https://docs.rs/gazebo/) [![Build status](https://img.shields.io/github/workflow/status/facebookincubator/gazebo/ci.svg)](https://github.com/facebookincubator/gazebo/actions) -This library contains a collection of well-tested utilities. Most modules stand alone, but taking a few representative examples: +This library contains a collection of well-tested utilities. Most modules stand +alone, but taking a few representative examples: -* `gazebo::prelude::*` is intended to be imported as such, and provides extension traits to common types. For example, it provides `Vec::map` which is equivalent to `iter().map(f).collect::>()`, and `str::split1` like `split` but which only splits once. We hope some of these functions one day make it into the Rust standard library. -* `gazebo::dupe` provides the trait `Dupe` with the member `dupe`, all of which are exactly like `Clone`. The difference is that `Dupe` should not be implemented for types that reallocate or have expensive `clone` operations - e.g. there is `Dupe` for `Arc` and `usize`, but not for `String` and `Vec`. By using `dupe` it is easy to focus on the `clone` calls (which should be rare) and ignore things whose cost is minimal. -* `gazebo::cell::ARef` provides a type which is either a `Ref` or a direct reference `&T`, with operations that make it look like `Ref` -- allowing you to uniformly convert a reference into something like a `Ref`. +- `gazebo::prelude::*` is intended to be imported as such, and provides + extension traits to common types. For example, it provides `Vec::map` which is + equivalent to `iter().map(f).collect::>()`, and `str::split1` like + `split` but which only splits once. We hope some of these functions one day + make it into the Rust standard library. +- `gazebo::dupe` provides the trait `Dupe` with the member `dupe`, all of which + are exactly like `Clone`. The difference is that `Dupe` should not be + implemented for types that reallocate or have expensive `clone` operations - + e.g. there is `Dupe` for `Arc` and `usize`, but not for `String` and `Vec`. By + using `dupe` it is easy to focus on the `clone` calls (which should be rare) + and ignore things whose cost is minimal. +- `gazebo::cell::ARef` provides a type which is either a `Ref` or a direct + reference `&T`, with operations that make it look like `Ref` -- allowing you + to uniformly convert a reference into something like a `Ref`. -The functionality provided by Gazebo is not stable, and continues to evolve with both additions (as we find new useful features) and removals (as we find better patterns or libraries encapsulating the ideas better). While the code varies in usefulness and design quality, it is all well tested and documented. +The functionality provided by Gazebo is not stable, and continues to evolve with +both additions (as we find new useful features) and removals (as we find better +patterns or libraries encapsulating the ideas better). While the code varies in +usefulness and design quality, it is all well tested and documented. ## Using Gazebo -Gazebo can be depended upon by adding `gazebo` to your `[dependencies]`, using the standard [Cargo patterns](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html). +Gazebo can be depended upon by adding `gazebo` to your `[dependencies]`, using +the standard +[Cargo patterns](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html). -The two interesting directories in this repo are `gazebo` (which contains the source to Gazebo itself) and `gazebo_derive` (which contains support for `#[derive(Dupe)]` and other Gazebo traits). Usually you will directly import `gazebo`, but `gazebo_derive` is a required transitive dependency if you are sourcing the library from GitHub. +The two interesting directories in this repo are `gazebo` (which contains the +source to Gazebo itself) and `gazebo_derive` (which contains support for +`#[derive(Dupe)]` and other Gazebo traits). Usually you will directly import +`gazebo`, but `gazebo_derive` is a required transitive dependency if you are +sourcing the library from GitHub. ## Learn More -You can learn more about Gazebo in [this introductory video](https://www.youtube.com/watch?v=pQJkx9HL_04), or from the following blog posts: +You can learn more about Gazebo in +[this introductory video](https://www.youtube.com/watch?v=pQJkx9HL_04), or from +the following blog posts: -* [Rust Nibbles - Gazebo: Prelude](https://developers.facebook.com/blog/post/2021/06/29/rust-nibbles-gazebo-prelude/) -* [Rust Nibbles - Gazebo: Dupe](https://developers.facebook.com/blog/post/2021/07/06/rust-nibbles-gazebo-dupe/) -* [Rust Nibbles - Gazebo: Variants](https://developers.facebook.com/blog/post/2021/07/13/rust-nibbles-gazebo-variants) -* [Rust Nibbles - Gazebo: AnyLifetime](https://developers.facebook.com/blog/post/2021/07/20/rust-nibbles-gazebo-any-lifetime/) -* [Rust Nibbles - Gazebo: Comparisons](https://developers.facebook.com/blog/post/2021/07/27/rust-nibbles-gazebo-comparisons/) -* [Rust Nibbles - Gazebo: Casts and Transmute](https://developers.facebook.com/blog/post/2021/08/03/rust-nibbles-gazebo-casts-transmute/) -* [Rust Nibbles - Gazebo: The rest of the tent](https://developers.facebook.com/blog/post/2021/08/10/rust-nibbles-gazebo-rest-of-tent/) +- [Rust Nibbles - Gazebo: Prelude](https://developers.facebook.com/blog/post/2021/06/29/rust-nibbles-gazebo-prelude/) +- [Rust Nibbles - Gazebo: Dupe](https://developers.facebook.com/blog/post/2021/07/06/rust-nibbles-gazebo-dupe/) +- [Rust Nibbles - Gazebo: Variants](https://developers.facebook.com/blog/post/2021/07/13/rust-nibbles-gazebo-variants) +- [Rust Nibbles - Gazebo: AnyLifetime](https://developers.facebook.com/blog/post/2021/07/20/rust-nibbles-gazebo-any-lifetime/) +- [Rust Nibbles - Gazebo: Comparisons](https://developers.facebook.com/blog/post/2021/07/27/rust-nibbles-gazebo-comparisons/) +- [Rust Nibbles - Gazebo: Casts and Transmute](https://developers.facebook.com/blog/post/2021/08/03/rust-nibbles-gazebo-casts-transmute/) +- [Rust Nibbles - Gazebo: The rest of the tent](https://developers.facebook.com/blog/post/2021/08/10/rust-nibbles-gazebo-rest-of-tent/) ## Making a release -1. Check the [GitHub Actions](https://github.com/facebookincubator/gazebo/actions) are green. -2. Update `CHANGELOG.md` with the changes since the last release. [This link](https://github.com/facebookincubator/gazebo/compare/v0.1.0...main) can help (update to compare against the last release). -3. Update the version numbers of the two `Cargo.toml` files. Bump them by 0.0.1 if there are no incompatible changes, or 0.1.0 if there are. Bump the dependency in `gazebo` to point at the latest `gazebo_derive` version. -4. Copy the files `CHANGELOG.md`, the two `LICENSE-` files and `README.md` into each `gazebo` and `gazebo_derive` subdirectory. -5. Run `cargo publish --allow-dirty --dry-run`, then without the `--dry-run`, first in `gazebo_derive` and then `gazebo` directories. -6. Create a [GitHub release](https://github.com/facebookincubator/gazebo/releases/new) with `v0.X.Y`, using the `gazebo` version as the name. +1. Check the + [GitHub Actions](https://github.com/facebookincubator/gazebo/actions) are + green. +2. Update `CHANGELOG.md` with the changes since the last release. + [This link](https://github.com/facebookincubator/gazebo/compare/v0.1.0...main) + can help (update to compare against the last release). +3. Update the version numbers of the two `Cargo.toml` files. Bump them by 0.0.1 + if there are no incompatible changes, or 0.1.0 if there are. Bump the + dependency in `gazebo` to point at the latest `gazebo_derive` version. +4. Copy the files `CHANGELOG.md`, the two `LICENSE-` files and `README.md` into + each `gazebo` and `gazebo_derive` subdirectory. +5. Run `cargo publish --allow-dirty --dry-run`, then without the `--dry-run`, + first in `gazebo_derive` and then `gazebo` directories. +6. Create a + [GitHub release](https://github.com/facebookincubator/gazebo/releases/new) + with `v0.X.Y`, using the `gazebo` version as the name. ## License -Gazebo is both MIT and Apache License, Version 2.0 licensed, as found in the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. +Gazebo is both MIT and Apache License, Version 2.0 licensed, as found in the +[LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. diff --git a/gazebo/display_container/src/lib.rs b/gazebo/display_container/src/lib.rs index d6a81993093dd..a413da544e8fa 100644 --- a/gazebo/display_container/src/lib.rs +++ b/gazebo/display_container/src/lib.rs @@ -13,17 +13,21 @@ //! //! ``` //! use std::fmt; +//! //! use display_container::*; //! //! struct MyItems(Vec<(String, i32)>); //! //! impl fmt::Display for MyItems { //! fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { -//! fmt_container(f, "{", "}", +//! fmt_container( +//! f, +//! "{", +//! "}", //! iter_display_chain( //! &["magic"], -//! self.0.iter().map(|(k, v)| display_pair(k, "=", v)) -//! ) +//! self.0.iter().map(|(k, v)| display_pair(k, "=", v)), +//! ), //! ) //! } //! } diff --git a/gazebo/dupe/src/iter.rs b/gazebo/dupe/src/iter.rs index 82ff76ea6b38c..9cd6431871db5 100644 --- a/gazebo/dupe/src/iter.rs +++ b/gazebo/dupe/src/iter.rs @@ -19,6 +19,7 @@ pub trait IterDupedExt: Sized { /// /// ``` /// use std::rc::Rc; + /// /// use dupe::IterDupedExt; /// let inputs = vec![Rc::new("Hello"), Rc::new("World")]; /// let outputs = inputs.iter().duped().collect::>(); diff --git a/gazebo/dupe/src/option.rs b/gazebo/dupe/src/option.rs index e1ac3db80c31b..899d8b42cd6ba 100644 --- a/gazebo/dupe/src/option.rs +++ b/gazebo/dupe/src/option.rs @@ -17,6 +17,7 @@ pub trait OptionDupedExt { /// /// ``` /// use std::rc::Rc; + /// /// use dupe::OptionDupedExt; /// let rc = Rc::new("test"); /// assert_eq!(Some(&rc).duped(), Some(rc)); diff --git a/gazebo/gazebo/src/cmp.rs b/gazebo/gazebo/src/cmp.rs index 2e459ead9318a..b405d269a2064 100644 --- a/gazebo/gazebo/src/cmp.rs +++ b/gazebo/gazebo/src/cmp.rs @@ -18,6 +18,7 @@ /// /// ``` /// use std::cmp::Ordering; +/// /// use gazebo::cmp_chain; /// /// assert_eq!( diff --git a/gazebo/gazebo/src/ext/iter.rs b/gazebo/gazebo/src/ext/iter.rs index 07afe98345fe0..2aa80b6eef5e7 100644 --- a/gazebo/gazebo/src/ext/iter.rs +++ b/gazebo/gazebo/src/ext/iter.rs @@ -20,11 +20,7 @@ pub trait IterExt { /// use gazebo::prelude::*; /// /// fn true_if_even_throw_on_zero(x: &usize) -> Result { - /// if *x == 0 { - /// Err(()) - /// } else { - /// Ok(x % 2 == 0) - /// } + /// if *x == 0 { Err(()) } else { Ok(x % 2 == 0) } /// } /// /// let x = [1, 3, 2]; @@ -35,7 +31,6 @@ pub trait IterExt { /// /// let x = [1, 0, 2]; /// assert_eq!(x.iter().try_any(true_if_even_throw_on_zero), Err(())); - /// /// ``` fn try_any(self, any: F) -> Result where @@ -49,11 +44,7 @@ pub trait IterExt { /// use gazebo::prelude::*; /// /// fn true_if_even_throw_on_zero(x: &usize) -> Result { - /// if *x == 0 { - /// Err(()) - /// } else { - /// Ok(x % 2 == 0) - /// } + /// if *x == 0 { Err(()) } else { Ok(x % 2 == 0) } /// } /// /// let x = [2, 4, 2]; @@ -64,7 +55,6 @@ pub trait IterExt { /// /// let x = [2, 0, 2]; /// assert_eq!(x.iter().try_all(true_if_even_throw_on_zero), Err(())); - /// /// ``` fn try_all(self, any: F) -> Result where @@ -105,9 +95,10 @@ pub trait IterExt { /// on the first encounter of `Err`. /// /// ``` - /// use gazebo::prelude::*; /// use std::cmp::Ordering; /// + /// use gazebo::prelude::*; + /// /// fn double_cmp_throw_on_zero(x: &usize, y: &usize) -> Result { /// if *x == 0 || *y == 0 { /// Err(()) @@ -119,27 +110,42 @@ pub trait IterExt { /// let x = [1, 4, 2]; /// let y = [2, 8, 4]; /// - /// assert_eq!(x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), Ok(Ordering::Equal)); + /// assert_eq!( + /// x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), + /// Ok(Ordering::Equal) + /// ); /// /// let x = [1, 2, 2]; /// let y = [2, 8, 4]; /// - /// assert_eq!(x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), Ok(Ordering::Less)); + /// assert_eq!( + /// x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), + /// Ok(Ordering::Less) + /// ); /// /// let x = [1, 4]; /// let y = [2, 8, 4]; /// - /// assert_eq!(x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), Ok(Ordering::Less)); + /// assert_eq!( + /// x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), + /// Ok(Ordering::Less) + /// ); /// /// let x = [1, 4, 4]; /// let y = [2, 8, 4]; /// - /// assert_eq!(x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), Ok(Ordering::Greater)); + /// assert_eq!( + /// x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), + /// Ok(Ordering::Greater) + /// ); /// /// let x = [1, 4, 2, 3]; /// let y = [2, 8, 4]; /// - /// assert_eq!(x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), Ok(Ordering::Greater)); + /// assert_eq!( + /// x.iter().try_cmp_by(&y, double_cmp_throw_on_zero), + /// Ok(Ordering::Greater) + /// ); /// /// let x = [1, 4, 2]; /// let y = [2, 0, 4]; @@ -164,7 +170,10 @@ pub trait IterExt { /// /// let i = vec![Ok((1, "a")), Err(()), Ok((2, "b"))]; /// - /// assert_eq!(i.into_iter().try_unzip::<_, _, Vec<_>, Vec<_>, _>(), Err(())); + /// assert_eq!( + /// i.into_iter().try_unzip::<_, _, Vec<_>, Vec<_>, _>(), + /// Err(()) + /// ); /// ``` fn try_unzip(self) -> Result<(FromA, FromB), E> where @@ -199,7 +208,10 @@ pub trait IterOwned: Sized { /// /// let inputs = vec!["a", "b", "c"]; /// let outputs = inputs.into_iter().owned().collect::>(); - /// assert_eq!(outputs, vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]) + /// assert_eq!( + /// outputs, + /// vec!["a".to_owned(), "b".to_owned(), "c".to_owned()] + /// ) /// ``` fn owned(self) -> Owned; } diff --git a/gazebo/gazebo/src/ext/vec.rs b/gazebo/gazebo/src/ext/vec.rs index 9baf1d267d503..b70b79ee6166d 100644 --- a/gazebo/gazebo/src/ext/vec.rs +++ b/gazebo/gazebo/src/ext/vec.rs @@ -44,8 +44,8 @@ pub trait SliceExt { /// /// ``` /// use gazebo::prelude::*; - /// assert_eq!([1,2,3][..].map(|x| x*x), vec![1,4,9]); - /// assert_eq!(vec![1,2,3].map(|x| x*x), vec![1,4,9]); + /// assert_eq!([1, 2, 3][..].map(|x| x * x), vec![1, 4, 9]); + /// assert_eq!(vec![1, 2, 3].map(|x| x * x), vec![1, 4, 9]); /// ``` /// /// Note that from Rust 1.55.0 there is a `map` method on @@ -59,8 +59,14 @@ pub trait SliceExt { /// /// ``` /// use gazebo::prelude::*; - /// assert_eq!([1,2,3].try_map(|x| Ok(x*x)), Ok::<_, bool>(vec![1,4,9])); - /// assert_eq!([1,2,-3].try_map(|x| if *x > 0 { Ok(x*x) } else { Err(false) }), Err(false)); + /// assert_eq!( + /// [1, 2, 3].try_map(|x| Ok(x * x)), + /// Ok::<_, bool>(vec![1, 4, 9]) + /// ); + /// assert_eq!( + /// [1, 2, -3].try_map(|x| if *x > 0 { Ok(x * x) } else { Err(false) }), + /// Err(false) + /// ); /// ``` /// /// This function will be generalised to [`Try`](std::ops::Try) once it has been @@ -162,12 +168,12 @@ impl SliceExt for [T] { /// struct X; /// /// let x = [&X]; -/// let y : Vec = x.cloned(); +/// let y: Vec = x.cloned(); /// /// assert_eq!(y, vec![X]); /// /// let x = vec![&X]; -/// let y : Vec = x.cloned(); +/// let y: Vec = x.cloned(); /// /// assert_eq!(y, vec![X]); /// ``` @@ -198,12 +204,12 @@ where /// struct X; /// /// let x = [&X]; -/// let y : Vec = x.duped(); +/// let y: Vec = x.duped(); /// /// assert_eq!(y, vec![X]); /// /// let x = vec![&X]; -/// let y : Vec = x.duped(); +/// let y: Vec = x.duped(); /// /// assert_eq!(y, vec![X]); /// ``` @@ -233,12 +239,12 @@ where /// struct X; /// /// let x = [&X]; -/// let y : Vec = x.copied(); +/// let y: Vec = x.copied(); /// /// assert_eq!(y, vec![X]); /// /// let x = vec![&X]; -/// let y : Vec = x.copied(); +/// let y: Vec = x.copied(); /// /// assert_eq!(y, vec![X]); /// ``` @@ -267,7 +273,7 @@ pub trait VecExt { /// /// ``` /// use gazebo::prelude::*; - /// assert_eq!(vec![1,2,3].into_map(|x| x*x), vec![1,4,9]); + /// assert_eq!(vec![1, 2, 3].into_map(|x| x * x), vec![1, 4, 9]); /// ``` fn into_map(self, f: F) -> Vec where @@ -277,8 +283,14 @@ pub trait VecExt { /// /// ``` /// use gazebo::prelude::*; - /// assert_eq!(vec![1,2,3].into_try_map(|x| Ok(x*x)), Ok::<_, bool>(vec![1,4,9])); - /// assert_eq!(vec![1,2,-3].into_try_map(|x| if x > 0 { Ok(x*x) } else { Err(false) }), Err(false)); + /// assert_eq!( + /// vec![1, 2, 3].into_try_map(|x| Ok(x * x)), + /// Ok::<_, bool>(vec![1, 4, 9]) + /// ); + /// assert_eq!( + /// vec![1, 2, -3].into_try_map(|x| if x > 0 { Ok(x * x) } else { Err(false) }), + /// Err(false) + /// ); /// ``` /// /// This function will be generalised to [`Try`](std::ops::Try) once it has been diff --git a/gazebo/gazebo/src/types.rs b/gazebo/gazebo/src/types.rs index b45c86faed15a..55bac92a7b627 100644 --- a/gazebo/gazebo/src/types.rs +++ b/gazebo/gazebo/src/types.rs @@ -18,8 +18,8 @@ /// /// ``` /// use gazebo::types::TEq; -/// fn foo>(x: A) -> String { -/// x.teq() +/// fn foo>(x: A) -> String { +/// x.teq() /// } /// ``` /// diff --git a/gazebo/gazebo/src/variants.rs b/gazebo/gazebo/src/variants.rs index b22f070536d90..8a00ab4fcafe6 100644 --- a/gazebo/gazebo/src/variants.rs +++ b/gazebo/gazebo/src/variants.rs @@ -88,7 +88,6 @@ pub use gazebo_derive::UnpackVariants; /// assert_eq!(Foo::Baz(1).variant_name(), "Baz"); /// assert_eq!(Foo::Qux { i: 1 }.variant_name(), "Qux"); /// ``` -/// pub use gazebo_derive::VariantName; pub trait VariantName { diff --git a/integrations/rust-project/src/buck.rs b/integrations/rust-project/src/buck.rs index d30d98ab0f37a..d7b5ba6c377e0 100644 --- a/integrations/rust-project/src/buck.rs +++ b/integrations/rust-project/src/buck.rs @@ -253,11 +253,12 @@ fn resolve_renamed_dependencies( // is resolved, switch to `$deps`. for (renamed_crate, dependency_target) in &info.named_deps { if let Some(entry) = target_index.get(dependency_target) { - trace!(old_name = ?entry.info.crate_name(), new_name = ?renamed_crate, "renamed crate"); + let new_name = renamed_crate.replace('-', "_"); + trace!(old_name = ?entry.info.crate_name(), new_name = new_name, "renamed crate"); // if the renamed dependency was encountered before, rename the existing `Dep` rather // than create a new one with a new name but the same index. While this duplication doesn't // seem to have any noticeable impact in limited testing, the behavior will be closer to - // that of Rusty and Cargo. + // that of Cargo. // // However, if the renamed dependency wasn't encountered before, we create a new `Dep` with // the new name. @@ -265,11 +266,11 @@ fn resolve_renamed_dependencies( // The primary invariant that is being upheld is that each index should // have one associated name. match deps.iter_mut().find(|dep| dep.crate_index == entry.index) { - Some(dep) => dep.name = renamed_crate.to_string(), + Some(dep) => dep.name = new_name, None => { let dep = Dep { crate_index: entry.index, - name: renamed_crate.to_string(), + name: new_name, }; deps.push(dep); } @@ -601,6 +602,7 @@ fn merge_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, deps: vec![], tests: vec![Target::new("//foo-unittest")], @@ -625,6 +627,7 @@ fn merge_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, deps: vec![Target::new("//foo")], tests: vec![], @@ -658,6 +661,7 @@ fn merge_target_multiple_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, deps: vec![Target::new("//foo@rust")], tests: vec![ @@ -685,6 +689,7 @@ fn merge_target_multiple_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, deps: vec![], tests: vec![ @@ -712,6 +717,7 @@ fn merge_target_multiple_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, // foo_test depends on foo, which is reasonable, but // we need to be careful when merging test @@ -739,6 +745,7 @@ fn merge_target_multiple_tests_no_cycles() { srcs: vec![], mapped_srcs: BTreeMap::new(), crate_name: None, + crate_dynamic: None, crate_root: None, deps: vec![Target::new("//test-framework")], tests: vec![], @@ -768,3 +775,72 @@ fn merge_target_multiple_tests_no_cycles() { "Test dependencies should only come from the foo@rust-unittest crate" ); } + +#[test] +fn named_deps_underscores() { + let mut target_index = BTreeMap::new(); + target_index.insert( + Target::new("//bar"), + TargetInfoEntry { + index: 0, + info: TargetInfo { + name: "bar".to_owned(), + label: "bar".to_owned(), + kind: Kind::Library, + edition: None, + srcs: vec![], + mapped_srcs: BTreeMap::new(), + crate_name: None, + crate_dynamic: None, + crate_root: None, + deps: vec![], + tests: vec![], + named_deps: BTreeMap::new(), + proc_macro: None, + features: vec![], + env: BTreeMap::new(), + source_folder: PathBuf::from("/tmp"), + project_relative_buildfile: PathBuf::from("bar/BUCK"), + in_workspace: false, + out_dir: None, + }, + }, + ); + + let mut named_deps = BTreeMap::new(); + named_deps.insert("bar-baz".to_owned(), Target::new("//bar")); + + let info = TargetInfo { + name: "foo".to_owned(), + label: "foo".to_owned(), + kind: Kind::Library, + edition: None, + srcs: vec![], + mapped_srcs: BTreeMap::new(), + crate_name: None, + crate_dynamic: None, + crate_root: None, + deps: vec![], + tests: vec![], + named_deps, + proc_macro: None, + features: vec![], + env: BTreeMap::new(), + source_folder: PathBuf::from("/tmp"), + project_relative_buildfile: PathBuf::from("foo/BUCK"), + in_workspace: false, + out_dir: None, + }; + + let mut deps = + resolve_dependencies_aliases(&info, &target_index, &BTreeMap::new(), &BTreeMap::new()); + resolve_renamed_dependencies(&info, &target_index, &mut deps); + + assert_eq!( + deps, + vec![Dep { + crate_index: 0, + name: "bar_baz".to_owned() + }] + ); +} diff --git a/integrations/rust-project/src/target.rs b/integrations/rust-project/src/target.rs index a6549efda7ea5..4499cef764c41 100644 --- a/integrations/rust-project/src/target.rs +++ b/integrations/rust-project/src/target.rs @@ -10,6 +10,7 @@ use std::collections::BTreeMap; use std::ffi::OsStr; use std::fmt; +use std::fs; use std::ops::Deref; use std::path::Path; use std::path::PathBuf; @@ -108,6 +109,7 @@ pub struct TargetInfo { pub mapped_srcs: BTreeMap, #[serde(rename = "crate")] pub crate_name: Option, + pub crate_dynamic: Option, pub crate_root: Option, #[serde(rename = "buck.deps", alias = "buck.direct_dependencies", default)] pub deps: Vec, @@ -139,6 +141,11 @@ pub struct TargetInfoEntry { impl TargetInfo { pub fn crate_name(&self) -> String { + if let Some(crate_dynamic) = &self.crate_dynamic { + if let Ok(contents) = fs::read_to_string(crate_dynamic) { + return contents.trim().to_owned(); + } + } self.crate_name.as_deref().map_or_else( || self.name.as_str().replace('-', "_"), |crate_name| crate_name.to_owned(), diff --git a/prelude/android/aapt2_link.bzl b/prelude/android/aapt2_link.bzl index 6a0ddea23a523..4044e2d3f88ac 100644 --- a/prelude/android/aapt2_link.bzl +++ b/prelude/android/aapt2_link.bzl @@ -33,9 +33,9 @@ def get_aapt2_link( link_infos = [] for use_proto_format in [False, True]: if use_proto_format: - identifier = "use_proto_format" + identifier = "use_proto" else: - identifier = "not_proto_format" + identifier = "not_proto" aapt2_command = cmd_args(android_toolchain.aapt2) aapt2_command.add("link") @@ -48,7 +48,7 @@ def get_aapt2_link( aapt2_command.add(["--proguard", proguard_config.as_output()]) # We don't need the R.java output, but aapt2 won't output R.txt unless we also request R.java. - r_dot_java = ctx.actions.declare_output("{}/initial-rdotjava".format(identifier), dir = True) + r_dot_java = ctx.actions.declare_output("{}/init-rjava".format(identifier), dir = True) aapt2_command.add(["--java", r_dot_java.as_output()]) r_dot_txt = ctx.actions.declare_output("{}/R.txt".format(identifier)) aapt2_command.add(["--output-text-symbols", r_dot_txt.as_output()]) diff --git a/prelude/android/android.bzl b/prelude/android/android.bzl index 48cec50fb89c1..7818bfae57082 100644 --- a/prelude/android/android.bzl +++ b/prelude/android/android.bzl @@ -5,6 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//:buck2_compatibility.bzl", "BUCK2_COMPATIBILITY_ATTRIB_NAME", "BUCK2_COMPATIBILITY_ATTRIB_TYPE") load("@prelude//android:cpu_filters.bzl", "ALL_CPU_FILTERS") load("@prelude//java:java.bzl", "AbiGenerationMode", "dex_min_sdk_version") load("@prelude//decls/android_rules.bzl", "AaptMode", "DuplicateResourceBehaviour") @@ -95,6 +96,7 @@ extra_attributes = { "_is_force_single_cpu": attrs.default_only(attrs.bool(default = FORCE_SINGLE_CPU)), "_is_force_single_default_cpu": attrs.default_only(attrs.bool(default = FORCE_SINGLE_DEFAULT_CPU)), "_java_toolchain": toolchains_common.java_for_android(), + BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, }, "android_build_config": { "_android_toolchain": toolchains_common.android(), @@ -123,6 +125,7 @@ extra_attributes = { "_is_force_single_cpu": attrs.default_only(attrs.bool(default = FORCE_SINGLE_CPU)), "_is_force_single_default_cpu": attrs.default_only(attrs.bool(default = FORCE_SINGLE_DEFAULT_CPU)), "_java_toolchain": toolchains_common.java_for_android(), + BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, }, "android_instrumentation_apk": { "aapt_mode": attrs.enum(AaptMode, default = "aapt1"), # Match default in V1 @@ -148,6 +151,7 @@ extra_attributes = { "_android_toolchain": toolchains_common.android(), "_exec_os_type": buck.exec_os_type_arg(), "_java_toolchain": toolchains_common.java_for_android(), + BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, }, "android_library": { "abi_generation_mode": attrs.option(attrs.enum(AbiGenerationMode), default = None), @@ -160,6 +164,7 @@ extra_attributes = { "_is_building_android_binary": is_building_android_binary_attr(), "_java_toolchain": toolchains_common.java_for_android(), "_kotlin_toolchain": toolchains_common.kotlin(), + BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, }, "android_manifest": { "_android_toolchain": toolchains_common.android(), @@ -187,6 +192,7 @@ extra_attributes = { "apk_genrule": genrule_attributes() | { "type": attrs.string(default = "apk"), "_android_toolchain": toolchains_common.android(), + "_exec_os_type": buck.exec_os_type_arg(), }, "gen_aidl": { "import_paths": attrs.list(attrs.arg(), default = []), @@ -201,6 +207,7 @@ extra_attributes = { "abi_generation_mode": attrs.option(attrs.enum(AbiGenerationMode), default = None), "resources_root": attrs.option(attrs.string(), default = None), "robolectric_runtime_dependencies": attrs.list(attrs.source(), default = []), + "test_class_names_file": attrs.option(attrs.source(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "_android_toolchain": toolchains_common.android(), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), @@ -209,5 +216,6 @@ extra_attributes = { "_java_test_toolchain": toolchains_common.java_test(), "_java_toolchain": toolchains_common.java_for_host_test(), "_kotlin_toolchain": toolchains_common.kotlin(), + BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, }, } diff --git a/prelude/android/android_aar.bzl b/prelude/android/android_aar.bzl index 1b07a4ed8ed62..b0c736f2ca51d 100644 --- a/prelude/android/android_aar.bzl +++ b/prelude/android/android_aar.bzl @@ -16,13 +16,16 @@ load("@prelude//android:cpu_filters.bzl", "CPU_FILTER_FOR_DEFAULT_PLATFORM", "CP load("@prelude//android:util.bzl", "create_enhancement_context") load("@prelude//java:java_providers.bzl", "get_all_java_packaging_deps", "get_all_java_packaging_deps_from_packaging_infos") load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") +load("@prelude//utils:set.bzl", "set") def android_aar_impl(ctx: AnalysisContext) -> list[Provider]: deps_by_platform = get_deps_by_platform(ctx) primary_platform = CPU_FILTER_FOR_PRIMARY_PLATFORM if CPU_FILTER_FOR_PRIMARY_PLATFORM in deps_by_platform else CPU_FILTER_FOR_DEFAULT_PLATFORM deps = deps_by_platform[primary_platform] - java_packaging_deps = [packaging_dep for packaging_dep in get_all_java_packaging_deps(ctx, deps)] + excluded_java_packaging_deps = get_all_java_packaging_deps(ctx, ctx.attrs.excluded_java_deps) + excluded_java_packaging_deps_targets = set([excluded_dep.label.raw_target() for excluded_dep in excluded_java_packaging_deps]) + java_packaging_deps = [packaging_dep for packaging_dep in get_all_java_packaging_deps(ctx, deps) if not excluded_java_packaging_deps_targets.contains(packaging_dep.label.raw_target())] android_packageable_info = merge_android_packageable_info(ctx.label, ctx.actions, deps) android_manifest = get_manifest(ctx, android_packageable_info, manifest_entries = {}) diff --git a/prelude/android/android_binary_native_library_rules.bzl b/prelude/android/android_binary_native_library_rules.bzl index 389da79fe9c8b..5dbaf94c34a73 100644 --- a/prelude/android/android_binary_native_library_rules.bzl +++ b/prelude/android/android_binary_native_library_rules.bzl @@ -60,9 +60,10 @@ load( "traverse_shared_library_info", ) load("@prelude//linking:strip.bzl", "strip_object") -load("@prelude//utils:graph_utils.bzl", "breadth_first_traversal_by", "post_order_traversal", "pre_order_traversal", "pre_order_traversal_by") +load("@prelude//utils:expect.bzl", "expect") +load("@prelude//utils:graph_utils.bzl", "breadth_first_traversal_by", "post_order_traversal", "pre_order_traversal") load("@prelude//utils:set.bzl", "set", "set_type") # @unused Used as a type -load("@prelude//utils:utils.bzl", "dedupe_by_value", "expect") +load("@prelude//utils:utils.bzl", "dedupe_by_value") # Native libraries on Android are built for a particular Application Binary Interface (ABI). We # package native libraries for one (or more, for multi-arch builds) ABIs into an Android APK. @@ -87,8 +88,8 @@ load("@prelude//utils:utils.bzl", "dedupe_by_value", "expect") # # Any native library that is not part of the root module (i.e. it is part of some other Voltron # module) is automatically packaged as an asset, and the assets for each module are compressed -# to a single `assets//libs.xz`. Similarly, the metadata for each module is stored -# at `assets//libs.txt`. +# to a single `assets//libs.xz` only if `compress_asset_libraries` is set to True. +# Similarly, the metadata for each module is stored at `assets//libs.txt`. def get_android_binary_native_library_info( enhance_ctx: EnhancementContext, @@ -137,7 +138,7 @@ def get_android_binary_native_library_info( root_module_metadata_assets = ctx.actions.declare_output("root_module_metadata_assets_symlink") root_module_compressed_lib_assets = ctx.actions.declare_output("root_module_compressed_lib_assets_symlink") non_root_module_metadata_assets = ctx.actions.declare_output("non_root_module_metadata_assets_symlink") - non_root_module_compressed_lib_assets = ctx.actions.declare_output("non_root_module_compressed_lib_assets_symlink") + non_root_module_lib_assets = ctx.actions.declare_output("non_root_module_lib_assets_symlink") unstripped_native_libraries = ctx.actions.declare_output("unstripped_native_libraries") unstripped_native_libraries_json = ctx.actions.declare_output("unstripped_native_libraries_json") @@ -155,7 +156,7 @@ def get_android_binary_native_library_info( root_module_metadata_assets, root_module_compressed_lib_assets, non_root_module_metadata_assets, - non_root_module_compressed_lib_assets, + non_root_module_lib_assets, ] fake_input = ctx.actions.write("dynamic.trigger", "") @@ -186,7 +187,7 @@ def get_android_binary_native_library_info( enable_relinker = getattr(ctx.attrs, "enable_relinker", False) if has_native_merging or enable_relinker: - native_merge_debug = ctx.actions.declare_output("native_merge.debug") + native_merge_debug = ctx.actions.declare_output("native_merge_debug", dir = True) dynamic_outputs.append(native_merge_debug) # We serialize info about the linkable graph and the apk module mapping and pass that to an @@ -344,7 +345,7 @@ def get_android_binary_native_library_info( ctx.actions.symlink_file(outputs[root_module_metadata_assets], dynamic_info.root_module_metadata_assets) ctx.actions.symlink_file(outputs[root_module_compressed_lib_assets], dynamic_info.root_module_compressed_lib_assets) ctx.actions.symlink_file(outputs[non_root_module_metadata_assets], dynamic_info.non_root_module_metadata_assets) - ctx.actions.symlink_file(outputs[non_root_module_compressed_lib_assets], dynamic_info.non_root_module_compressed_lib_assets) + ctx.actions.symlink_file(outputs[non_root_module_lib_assets], dynamic_info.non_root_module_lib_assets) ctx.actions.dynamic_output(dynamic = dynamic_inputs, inputs = [], outputs = dynamic_outputs, f = dynamic_native_libs_info) all_native_libs = ctx.actions.symlinked_dir("debug_all_native_libs", {"others": native_libs, "primary": native_libs_always_in_primary_apk}) @@ -363,7 +364,7 @@ def get_android_binary_native_library_info( native_libs_for_primary_apk = native_libs_for_primary_apk, exopackage_info = exopackage_info, root_module_native_lib_assets = [native_lib_assets_for_primary_apk, stripped_native_linkable_assets_for_primary_apk, root_module_metadata_assets, root_module_compressed_lib_assets], - non_root_module_native_lib_assets = [non_root_module_metadata_assets, non_root_module_compressed_lib_assets], + non_root_module_native_lib_assets = [non_root_module_metadata_assets, non_root_module_lib_assets], generated_java_code = generated_java_code, ) @@ -380,7 +381,7 @@ _NativeLibsAndAssetsInfo = record( root_module_metadata_assets = Artifact, root_module_compressed_lib_assets = Artifact, non_root_module_metadata_assets = Artifact, - non_root_module_compressed_lib_assets = Artifact, + non_root_module_lib_assets = Artifact, ) def _get_exopackage_info( @@ -464,6 +465,7 @@ def _get_native_libs_and_assets( root_module_compressed_lib_srcs = {} non_root_module_metadata_srcs = {} non_root_module_compressed_lib_srcs = {} + non_root_module_uncompressed_libs = [] assets_for_primary_apk = filter(None, [native_lib_assets_for_primary_apk, stripped_linkables.linkable_assets_for_primary_apk]) stripped_linkable_assets_for_primary_apk = stripped_linkables.linkable_assets_for_primary_apk if assets_for_primary_apk: @@ -480,8 +482,26 @@ def _get_native_libs_and_assets( for module, native_lib_assets in native_lib_module_assets_map.items(): metadata_file, native_library_paths = _get_native_libs_as_assets_metadata(ctx, native_lib_assets, module) non_root_module_metadata_srcs[paths.join(_get_native_libs_as_assets_dir(module), "libs.txt")] = metadata_file - compressed_lib_dir = _get_compressed_native_libs_as_assets(ctx, native_lib_assets, native_library_paths, module) - non_root_module_compressed_lib_srcs[_get_native_libs_as_assets_dir(module)] = compressed_lib_dir + if ctx.attrs.compress_asset_libraries: + compressed_lib_dir = _get_compressed_native_libs_as_assets(ctx, native_lib_assets, native_library_paths, module) + non_root_module_compressed_lib_srcs[_get_native_libs_as_assets_dir(module)] = compressed_lib_dir + else: + non_root_module_uncompressed_libs.extend(native_lib_assets) + + if non_root_module_uncompressed_libs: + expect(not non_root_module_compressed_lib_srcs, "Cannot have both uncompressed and compressed native libraries for a non-root module") + non_root_module_libs = ctx.actions.declare_output("non_root_module_libs") + ctx.actions.run( + cmd_args([ + ctx.attrs._android_toolchain[AndroidToolchainInfo].combine_native_library_dirs[RunInfo], + "--output-dir", + non_root_module_libs.as_output(), + "--library-dirs", + ] + non_root_module_uncompressed_libs), + category = "combine_non_root_module_native_libs", + ) + else: + non_root_module_libs = ctx.actions.symlinked_dir("non_root_module_libs", non_root_module_compressed_lib_srcs) combined_native_libs = ctx.actions.declare_output("combined_native_libs", dir = True) native_libs_metadata = ctx.actions.declare_output("native_libs_metadata.txt") @@ -515,7 +535,7 @@ def _get_native_libs_and_assets( root_module_metadata_assets = ctx.actions.symlinked_dir("root_module_metadata_assets", root_module_metadata_srcs), root_module_compressed_lib_assets = ctx.actions.symlinked_dir("root_module_compressed_lib_assets", root_module_compressed_lib_srcs), non_root_module_metadata_assets = ctx.actions.symlinked_dir("non_root_module_metadata_assets", non_root_module_metadata_srcs), - non_root_module_compressed_lib_assets = ctx.actions.symlinked_dir("non_root_module_compressed_lib_assets", non_root_module_compressed_lib_srcs), + non_root_module_lib_assets = non_root_module_libs, ) def _filter_prebuilt_native_library_dir( @@ -680,6 +700,9 @@ def encode_linkable_graph_for_mergemap(graph_node_map_by_platform: dict[str, dic platform: { target: _LinkableSharedNode( raw_target = str(target.raw_target()), + # FIXME(JakobDegen): The definition of `LinkableNode` claims that it's ok for this + # to be `None` (I assume in the case of static preferred linkage), so either that is + # wrong or this is. See the diff that added this FIXME for how to reproduce soname = node.default_soname, labels = node.labels, deps = node.deps + node.exported_deps, @@ -734,16 +757,25 @@ LinkGroupData = record( apk_module = str, ) +# Lookup key for somerge groups, either the soname for shared libraries or the target name for unmerged statics +GroupLabel = str + +# Represents the primary constituents and deps of primary constituents used to create a LinkGroupLinkableNode for a non-prebuilt shared library. +LinkGroupMergeInfo = record( + label = GroupLabel, + deps = list[GroupLabel], + exported_deps = list[GroupLabel], + constituent_link_infos = list[LinkInfo], +) + # Represents a node in the final merged linkable map. Most of these will be shared libraries, either prebuilt shared libs or -# libraries that are created below for a node in the link_groups_graph. The exception is for non-merged static-only nodes, in -# that case this +# libraries that are created below for a node in the link_groups_graph. The exception is for non-merged static-only nodes. LinkGroupLinkableNode = record( # The LinkInfo to add to the link line for a node that links against this. link = LinkInfo, - deps = list[str], - exported_deps = list[str], + deps = list[GroupLabel], + exported_deps = list[GroupLabel], shared_lib = [SharedLibrary, None], - # linker flags to be exported by any node that links against this. This can only be non-None for non-merged static only nodes (as we don't # propagate exported linker flags through transitive shared lib deps). exported_linker_flags = [(list[typing.Any], list[typing.Any]), None], @@ -999,12 +1031,16 @@ def _get_merged_linkables( for group in post_order_traversal(link_groups_graph): group_data = link_groups[group] is_actually_merged = len(group_data.constituents) > 1 + can_be_asset = True + for target in group_data.constituents: + if not linkable_nodes[target].can_be_asset: + can_be_asset = False + break if not is_actually_merged: target = group_data.constituents[0] node_data = linkable_nodes[target] - can_be_asset = node_data.can_be_asset if node_data.preferred_linkage == Linkage("static") or not _has_linkable(node_data): debug_info.unmerged_statics.append(target) @@ -1044,147 +1080,75 @@ def _get_merged_linkables( ) continue - # Keys in the current group stay as a Label, deps get converted to the group key. - def convert_to_merged_graph_deps(deps: list[Label], curr_group: str) -> list[[Label, str]]: - converted = [] - for dep in deps: - dep_group = target_to_link_group[dep] - if dep_group == curr_group: - converted.append(dep) - elif dep_group: - converted.append(dep_group) - return dedupe_by_value(converted) - - # For the current group, this will traverse the original linkable graph to find the LinkableNodes for - # the constituents of the group and traverses the link_group graph for non-constituent deps. - def get_merged_graph_traversal(curr_group: str, exported_only: bool) -> typing.Callable: - def traversal(key: [Label, str]) -> list[[Label, str]]: - if eval_type(Label).matches(key): - expect(target_to_link_group[key] == curr_group) - node = linkable_nodes[key] - if exported_only: - return convert_to_merged_graph_deps(node.exported_deps, curr_group) - return convert_to_merged_graph_deps(node.deps + node.exported_deps, curr_group) - else: - link_group_node = link_group_linkable_nodes[key] - if exported_only: - return link_group_node.exported_deps - return dedupe_by_value(link_group_node.deps + link_group_node.exported_deps) - - # It's easy for us to accidentally get this merged traversal wrong, so this provides one guardrail - def checked_traversal(key: [Label, str]) -> list[[Label, str]]: - return expect_dedupe(traversal(key)) - - return checked_traversal - - # note that this will possibly contain shared lib dependencies which aren't really public. that's handled below. - public_node_roots = group_data.constituents - - # this is a hybrid of buck1 somerge behavior and what we do for link groups. - # like link groups, we expose link group by setting link_whole on its link infos (this matches buck1 for - # primary constituents, but not for other constituents). - # like buck1, we treat all primary constituents as public node roots (as opposed to link groups that only treats - # preferred_linkage=shared and edges with an outbound dep as public roots), and then traverse exported deps from - # those roots to find all public nodes. - # the main thing to note from this is that for non-primary constituents that are identified as public, we will - # use link_whole whereas buck1 will make dependents link against them directly - exported_public_nodes = { - d: True - for d in breadth_first_traversal_by( - None, - public_node_roots, - get_merged_graph_traversal(group, True), - ) - } - exported_linker_flags = [] exported_linker_post_flags = [] links = [] - shared_lib_deps = [] - real_constituents = [] if is_actually_merged and merge_data.glue_linkable: - real_constituents.append(merge_data.glue_linkable[0]) links.append(set_link_info_link_whole(merge_data.glue_linkable[1])) solib_constituents = [] - link_group_deps = [] - ordered_group_constituents = pre_order_traversal_by(group_data.constituents, get_merged_graph_traversal(group, False)) - representative_label = ordered_group_constituents[0] - for key in ordered_group_constituents: - real_constituents.append(key) - if eval_type(Label).matches(key): - # This is handling targets within this link group - expect(target_to_link_group[key] == group) - node = linkable_nodes[key] - - default_solibs = list(node.shared_libs.keys()) - if not default_solibs and node.preferred_linkage == Linkage("static"): - default_solibs = [node.default_soname] - - for soname in default_solibs: - included_default_solibs[soname] = True - if node.include_in_android_mergemap: - solib_constituents.append(soname) - - node = linkable_nodes[key] - link_info = node.link_infos[archive_output_style].default - - # the propagated link info should already be wrapped with exported flags. - link_info = wrap_link_info( - link_info, - pre_flags = node.linker_flags.flags, - post_flags = node.linker_flags.post_flags, - ) - exported_linker_flags.extend(node.linker_flags.exported_flags) - exported_linker_post_flags.extend(node.linker_flags.exported_post_flags) - if key in exported_public_nodes: - link_info = set_link_info_link_whole(link_info) - else: - # This is cross-link-group deps. We add information to the link line from the LinkGroupLinkableNode of the dep. - link_group_node = link_group_linkable_nodes[key] - link_info = link_group_node.link - if link_group_node.shared_lib: - shared_lib_deps.append(link_group_node.shared_lib.soname) - link_group_deps.append(key) - elif key in exported_public_nodes: - link_info = set_link_info_link_whole(link_info) + group_deps = [] + group_exported_deps = [] + for key in group_data.constituents: + expect(target_to_link_group[key] == group) + node = linkable_nodes[key] + + default_solibs = list(node.shared_libs.keys()) + if not default_solibs and node.preferred_linkage == Linkage("static"): + default_solibs = [node.default_soname] + + for soname in default_solibs: + included_default_solibs[soname] = True + if node.include_in_android_mergemap: + solib_constituents.append(soname) + + node = linkable_nodes[key] + link_info = node.link_infos[archive_output_style].default + + # the propagated link info should already be wrapped with exported flags. + link_info = wrap_link_info( + link_info, + pre_flags = node.linker_flags.flags, + post_flags = node.linker_flags.post_flags, + ) + exported_linker_flags.extend(node.linker_flags.exported_flags) + exported_linker_post_flags.extend(node.linker_flags.exported_post_flags) + links.append(set_link_info_link_whole(link_info)) - if link_group_node.exported_linker_flags: - exported_linker_flags.extend(link_group_node.exported_linker_flags[0]) - exported_linker_post_flags.extend(link_group_node.exported_linker_flags[1]) + dep_groups = [target_to_link_group[dep] for dep in node.deps] + group_deps.extend([dep_group for dep_group in dep_groups if dep_group != group]) - links.append(link_info) + exported_dep_groups = [target_to_link_group[dep] for dep in node.exported_deps] + group_exported_deps.extend([dep_group for dep_group in exported_dep_groups if dep_group != group]) soname = group if not is_actually_merged: soname = linkable_nodes[group_data.constituents[0]].default_soname debug_info.with_default_soname.append((soname, group_data.constituents[0])) - debug_info.group_debug.setdefault( - group, - struct( - soname = soname, - merged = is_actually_merged, - constituents = real_constituents, - shlib_deps = shared_lib_deps, - exported_public_nodes = exported_public_nodes, - exported_linker_flags = exported_linker_flags, - exported_linker_post_flags = exported_linker_post_flags, - ), - ) - output_path = _platform_output_path(soname, platform if len(merged_data_by_platform) > 1 else None) - link_args = [LinkArgs(infos = links)] + + link_merge_info = LinkGroupMergeInfo( + label = group, + deps = dedupe_by_value(group_deps), + exported_deps = dedupe_by_value(group_exported_deps), + constituent_link_infos = links, + ) + link_args, shlib_deps, link_deps_graph = _create_merged_link_args( + root_target = link_merge_info, + linkable_nodes = link_group_linkable_nodes, + cxx_toolchain = cxx_toolchain, + ) shared_lib = create_shared_lib( ctx, output_path = output_path, soname = soname, - link_args = link_args, + link_args = [link_args], cxx_toolchain = cxx_toolchain, - shared_lib_deps = shared_lib_deps, - label = representative_label, + shared_lib_deps = [link_group_linkable_nodes[label].shared_lib.soname for label in shlib_deps], + label = group_data.constituents[0], can_be_asset = can_be_asset, ) @@ -1197,8 +1161,8 @@ def _get_merged_linkables( )], post_flags = exported_linker_post_flags, ), - deps = link_group_deps, - exported_deps = [], + deps = link_merge_info.deps, + exported_deps = link_merge_info.exported_deps, shared_lib = shared_lib, # exported linker flags for shared libs are in their linkinfo itself and are not exported from dependents exported_linker_flags = None, @@ -1211,6 +1175,19 @@ def _get_merged_linkables( is_actually_merged = is_actually_merged, ) + debug_info.group_debug.setdefault( + group, + struct( + soname = soname, + merged = is_actually_merged, + primary_constituents = group_data.constituents, + real_constituents = link_deps_graph.keys(), + shlib_deps = shlib_deps, + exported_linker_flags = exported_linker_flags, + exported_linker_post_flags = exported_linker_post_flags, + ), + ) + shared_libs_by_platform[platform] = group_shared_libs debug_info.missing_default_solibs.extend([d for d in merge_data.default_shared_libs if d not in included_default_solibs]) @@ -1423,6 +1400,52 @@ def _create_link_args( shlib_deps.append(target) else: links.append(node.link_infos[preferred_linkable_type].default) + + extra_runtime_flags = cxx_toolchain.linker_info.shared_dep_runtime_ld_flags or [] + if extra_runtime_flags: + links.append(LinkInfo(pre_flags = extra_runtime_flags)) + return LinkArgs(infos = links), shlib_deps, link_traversal_cache + +# Equivalent to _create_link_args but for somerge +def _create_merged_link_args( + *, + cxx_toolchain: CxxToolchainInfo, + root_target: LinkGroupMergeInfo, + linkable_nodes: dict[GroupLabel, LinkGroupLinkableNode]) -> (LinkArgs, list[GroupLabel], dict[GroupLabel, list[GroupLabel]]): + if LinkOrdering(cxx_toolchain.linker_info.link_ordering) != LinkOrdering("topological"): + fail("don't yet support link ordering {}".format(cxx_toolchain.linker_info.link_ordering)) + + link_traversal_cache = {} + + def link_traversal(label: GroupLabel) -> list[GroupLabel]: + def link_traversal_deps(label: GroupLabel): + if label == root_target.label: + return root_target.deps + root_target.exported_deps + + linkable_node = linkable_nodes[label] + if linkable_node.shared_lib: + return linkable_node.exported_deps + else: + return linkable_node.deps + linkable_node.exported_deps + + res = link_traversal_cache.get(label, None) + if res: + return res + res = link_traversal_deps(label) + link_traversal_cache[label] = res + return res + + links = [] + shlib_deps = [] + for label in _rust_matching_topological_traversal([root_target.label], link_traversal): + if label == root_target.label: + links.extend(root_target.constituent_link_infos) + else: + linkable_node = linkable_nodes[label] + links.append(linkable_node.link) + if linkable_node.shared_lib: + shlib_deps.append(label) + extra_runtime_flags = cxx_toolchain.linker_info.shared_dep_runtime_ld_flags or [] if extra_runtime_flags: links.append(LinkInfo(pre_flags = extra_runtime_flags)) diff --git a/prelude/android/android_instrumentation_test.bzl b/prelude/android/android_instrumentation_test.bzl index 27d56c21f7724..f14709ac30e00 100644 --- a/prelude/android/android_instrumentation_test.bzl +++ b/prelude/android/android_instrumentation_test.bzl @@ -14,6 +14,8 @@ load("@prelude//utils:expect.bzl", "expect") load("@prelude//test/inject_test_run_info.bzl", "inject_test_run_info") DEFAULT_ANDROID_SUBPLATFORM = "android-30" +DEFAULT_ANDROID_PLATFORM = "android-emulator" +DEFAULT_ANDROID_INSTRUMENTATION_TESTS_USE_CASE = "instrumentation-tests" def android_instrumentation_test_impl(ctx: AnalysisContext): android_toolchain = ctx.attrs._android_toolchain[AndroidToolchainInfo] @@ -91,10 +93,10 @@ def android_instrumentation_test_impl(ctx: AnalysisContext): local_enabled = android_toolchain.instrumentation_test_can_run_locally, remote_enabled = True, remote_execution_properties = { - "platform": "android-emulator", - "subplatform": _compute_emulator_target(ctx.attrs.labels or []), + "platform": _compute_emulator_platform(ctx.attrs.labels or []), + "subplatform": _compute_emulator_subplatform(ctx.attrs.labels or []), }, - remote_execution_use_case = "instrumentation-tests", + remote_execution_use_case = _compute_re_use_case(ctx.attrs.labels or []), ), "static-listing": CommandExecutorConfig( local_enabled = True, @@ -117,10 +119,26 @@ def android_instrumentation_test_impl(ctx: AnalysisContext): ] + classmap_source_info # replicating the logic in https://fburl.com/code/1fqowxu4 to match buck1's behavior -def _compute_emulator_target(labels: list[str]) -> str: - emulator_target_labels = [label for label in labels if label.startswith("re_emulator_")] - expect(len(emulator_target_labels) <= 1, "multiple 're_emulator_' labels were found:[{}], there must be only one!".format(", ".join(emulator_target_labels))) - if len(emulator_target_labels) == 0: +def _compute_emulator_subplatform(labels: list[str]) -> str: + emulator_subplatform_labels = [label for label in labels if label.startswith("re_emulator_")] + expect(len(emulator_subplatform_labels) <= 1, "multiple 're_emulator_' labels were found:[{}], there must be only one!".format(", ".join(emulator_subplatform_labels))) + if len(emulator_subplatform_labels) == 0: return DEFAULT_ANDROID_SUBPLATFORM - else: # len(emulator_target_labels) == 1: - return emulator_target_labels[0].replace("re_emulator_", "") + else: # len(emulator_subplatform_labels) == 1: + return emulator_subplatform_labels[0].replace("re_emulator_", "") + +def _compute_emulator_platform(labels: list[str]) -> str: + emulator_platform_labels = [label for label in labels if label.startswith("re_platform_")] + expect(len(emulator_platform_labels) <= 1, "multiple 're_platform_' labels were found:[{}], there must be only one!".format(", ".join(emulator_platform_labels))) + if len(emulator_platform_labels) == 0: + return DEFAULT_ANDROID_PLATFORM + else: # len(emulator_platform_labels) == 1: + return emulator_platform_labels[0].replace("re_platform_", "") + +def _compute_re_use_case(labels: list[str]) -> str: + re_use_case_labels = [label for label in labels if label.startswith("re_opts_use_case=")] + expect(len(re_use_case_labels) <= 1, "multiple 're_opts_use_case' labels were found:[{}], there must be only one!".format(", ".join(re_use_case_labels))) + if len(re_use_case_labels) == 0: + return DEFAULT_ANDROID_INSTRUMENTATION_TESTS_USE_CASE + else: # len(re_use_case_labels) == 1: + return re_use_case_labels[0].replace("re_opts_use_case=", "") diff --git a/prelude/android/android_prebuilt_aar.bzl b/prelude/android/android_prebuilt_aar.bzl index 1f7982a4dabd4..f0fb0f0e64206 100644 --- a/prelude/android/android_prebuilt_aar.bzl +++ b/prelude/android/android_prebuilt_aar.bzl @@ -104,7 +104,15 @@ def android_prebuilt_aar_impl(ctx: AnalysisContext) -> list[Provider]: linkable_graph, template_placeholder_info, java_library_intellij_info, - merge_android_packageable_info(ctx.label, ctx.actions, ctx.attrs.deps, manifest = manifest, prebuilt_native_library_dir = native_library, resource_info = resource_info), + merge_android_packageable_info( + ctx.label, + ctx.actions, + ctx.attrs.deps, + manifest = manifest, + prebuilt_native_library_dir = native_library, + resource_info = resource_info, + for_primary_apk = ctx.attrs.for_primary_apk, + ), resource_info, DefaultInfo(default_output = all_classes_jar, other_outputs = [ manifest, diff --git a/prelude/android/android_providers.bzl b/prelude/android/android_providers.bzl index 3299b6875dcfe..d93c12a3ced5e 100644 --- a/prelude/android/android_providers.bzl +++ b/prelude/android/android_providers.bzl @@ -160,6 +160,7 @@ ResourceInfoTSet = transitive_set() DepsInfo = record( name = TargetLabel, deps = list[TargetLabel], + for_primary_apk = bool, ) AndroidPackageableInfo = provider( @@ -244,7 +245,8 @@ def merge_android_packageable_info( build_config_info: [AndroidBuildConfigInfo, None] = None, manifest: [Artifact, None] = None, prebuilt_native_library_dir: [PrebuiltNativeLibraryDir, None] = None, - resource_info: [AndroidResourceInfo, None] = None) -> AndroidPackageableInfo: + resource_info: [AndroidResourceInfo, None] = None, + for_primary_apk: bool = False) -> AndroidPackageableInfo: android_packageable_deps = filter(None, [x.get(AndroidPackageableInfo) for x in deps]) build_config_infos = _get_transitive_set( @@ -260,6 +262,7 @@ def merge_android_packageable_info( DepsInfo( name = label.raw_target(), deps = [dep.target_label for dep in android_packageable_deps], + for_primary_apk = for_primary_apk, ), AndroidDepsTSet, ) diff --git a/prelude/android/configuration.bzl b/prelude/android/configuration.bzl index ea18b6d51fa38..f383e99f0c609 100644 --- a/prelude/android/configuration.bzl +++ b/prelude/android/configuration.bzl @@ -23,17 +23,16 @@ load("@prelude//utils:expect.bzl", "expect") # platforms). We only use the "arm64" native libraries if it is one of the specified platforms. We # "throw away" the non-native libraries for all other configured sub-graphs. +_DEFAULT_PLATFORM = "config//platform/android:x86_32-fbsource" + _REFS = { "arm64": "config//cpu/constraints:arm64", "armv7": "config//cpu/constraints:arm32", "build_only_native_code": "prelude//android/constraints:build_only_native_code", "building_android_binary": "prelude//os:building_android_binary", "cpu": "config//cpu/constraints:cpu", - "default_platform": "config//platform/android:x86_32-fbsource", "maybe_build_only_native_code": "prelude//android/constraints:maybe_build_only_native_code", "maybe_building_android_binary": "prelude//os:maybe_building_android_binary", - "maybe_merge_native_libraries": "config//features/android/constraints:maybe_merge_native_libraries", - "merge_native_libraries": "config//features/android/constraints:merge_native_libraries", "min_sdk_version": "prelude//android/constraints:min_sdk_version", "x86": "config//cpu/constraints:x86_32", "x86_64": "config//cpu/constraints:x86_64", @@ -42,6 +41,8 @@ for min_sdk in get_min_sdk_version_range(): constraint_value_name = get_min_sdk_version_constraint_value_name(min_sdk) _REFS[constraint_value_name] = "prelude//android/constraints:{}".format(constraint_value_name) +_REFS["default_platform"] = read_root_config("build", "default_platform", _DEFAULT_PLATFORM) + def _cpu_split_transition_impl( platform: PlatformInfo, refs: struct, @@ -57,17 +58,13 @@ def _cpu_split_transition_impl( refs, cpu_filters, attrs.min_sdk_version, - attrs.native_library_merge_map, - attrs.native_library_merge_sequence, ) def _cpu_split_transition( platform: PlatformInfo, refs: struct, cpu_filters: list[str], - min_sdk_version: [int, None], - native_library_merge_map: [dict[str, list[str]], None], - native_library_merge_sequence: [list[list[tuple] | tuple], None]) -> dict[str, PlatformInfo]: + min_sdk_version: [int, None]) -> dict[str, PlatformInfo]: cpu = refs.cpu x86 = refs.x86[ConstraintValueInfo] x86_64 = refs.x86_64[ConstraintValueInfo] @@ -101,9 +98,6 @@ def _cpu_split_transition( base_constraints[refs.maybe_building_android_binary[ConstraintSettingInfo].label] = refs.building_android_binary[ConstraintValueInfo] - if native_library_merge_map or native_library_merge_sequence: - base_constraints[refs.maybe_merge_native_libraries[ConstraintSettingInfo].label] = refs.merge_native_libraries[ConstraintValueInfo] - if min_sdk_version: base_constraints[refs.min_sdk_version[ConstraintSettingInfo].label] = _get_min_sdk_constraint_value(min_sdk_version, refs) @@ -136,8 +130,6 @@ cpu_split_transition = transition( attrs = [ "cpu_filters", "min_sdk_version", - "native_library_merge_map", - "native_library_merge_sequence", "_is_force_single_cpu", "_is_force_single_default_cpu", ], @@ -157,8 +149,6 @@ cpu_transition = transition( attrs = [ "cpu_filters", "min_sdk_version", - "native_library_merge_map", - "native_library_merge_sequence", "_is_force_single_cpu", "_is_force_single_default_cpu", ], diff --git a/prelude/android/dex_rules.bzl b/prelude/android/dex_rules.bzl index af03ea65d750c..02184f174458e 100644 --- a/prelude/android/dex_rules.bzl +++ b/prelude/android/dex_rules.bzl @@ -365,8 +365,7 @@ def _filter_pre_dexed_libs( batch_number: int) -> DexInputsWithClassNamesAndWeightEstimatesFile: weight_estimate_and_filtered_class_names_file = actions.declare_output("class_names_and_weight_estimates_for_batch_{}".format(batch_number)) - filter_dex_cmd = cmd_args([ - android_toolchain.filter_dex_class_names[RunInfo], + filter_dex_cmd_args = cmd_args([ "--primary-dex-patterns", primary_dex_patterns_file, "--dex-target-identifiers", @@ -378,6 +377,12 @@ def _filter_pre_dexed_libs( "--output", weight_estimate_and_filtered_class_names_file.as_output(), ]) + filter_dex_cmd_argsfile = actions.write("filter_dex_cmd_args_{}".format(batch_number), filter_dex_cmd_args) + + filter_dex_cmd = cmd_args([ + android_toolchain.filter_dex_class_names[RunInfo], + cmd_args(filter_dex_cmd_argsfile, format = "@{}").hidden(filter_dex_cmd_args), + ]) actions.run(filter_dex_cmd, category = "filter_dex", identifier = "batch_{}".format(batch_number)) return DexInputsWithClassNamesAndWeightEstimatesFile(libs = pre_dexed_libs, weight_estimate_and_filtered_class_names_file = weight_estimate_and_filtered_class_names_file) diff --git a/prelude/android/tools/combine_native_library_dirs.py b/prelude/android/tools/combine_native_library_dirs.py index a7aa7e58979bc..bbb52597e9b7a 100644 --- a/prelude/android/tools/combine_native_library_dirs.py +++ b/prelude/android/tools/combine_native_library_dirs.py @@ -51,8 +51,12 @@ def main() -> None: lib, ) - output_path.parent.mkdir(exist_ok=True) - output_path.symlink_to(os.readlink(lib)) + output_path.parent.mkdir(exist_ok=True, parents=True) + relative_path_to_lib = os.path.relpath( + os.path.realpath(lib), + start=os.path.realpath(os.path.dirname(output_path)), + ) + output_path.symlink_to(relative_path_to_lib) if args.metadata_file: with open(lib, "rb") as f: diff --git a/prelude/android/tools/filter_dex.py b/prelude/android/tools/filter_dex.py index e26d507e0adab..808f586e218da 100644 --- a/prelude/android/tools/filter_dex.py +++ b/prelude/android/tools/filter_dex.py @@ -72,7 +72,8 @@ def class_name_matches_filter(self, class_name): def _parse_args(): parser = argparse.ArgumentParser( - description="Tool to filter a dex for primary class names." + description="Tool to filter a dex for primary class names.", + fromfile_prefix_chars="@", ) parser.add_argument( diff --git a/prelude/android/voltron.bzl b/prelude/android/voltron.bzl index 5e8f697bbf45f..d6622c261a028 100644 --- a/prelude/android/voltron.bzl +++ b/prelude/android/voltron.bzl @@ -16,6 +16,7 @@ load( "traverse_shared_library_info", ) load("@prelude//utils:expect.bzl", "expect") +load("@prelude//utils:set.bzl", "set") load("@prelude//utils:utils.bzl", "flatten") # "Voltron" gives us the ability to split our Android APKs into different "modules". These @@ -149,11 +150,14 @@ def _get_base_cmd_and_output( application_module_dependencies: [dict[str, list[str]], None], application_module_blocklist: [list[list[Dependency]], None]) -> (cmd_args, Artifact): deps_map = {} + primary_apk_deps = set() for android_packageable_info in android_packageable_infos: if android_packageable_info.deps: for deps_info in android_packageable_info.deps.traverse(): deps = deps_map.setdefault(deps_info.name, []) deps_map[deps_info.name] = dedupe(deps + deps_info.deps) + if deps_info.for_primary_apk: + primary_apk_deps.add(deps_info.name) target_graph_file = actions.write_json("target_graph.json", deps_map) application_module_configs_map = { @@ -183,8 +187,8 @@ def _get_base_cmd_and_output( used_by_wrap_script_libs = [str(shared_lib.label.raw_target()) for shared_lib in shared_libraries if shared_lib.for_primary_apk] prebuilt_native_library_dirs = flatten([list(android_packageable_info.prebuilt_native_library_dirs.traverse()) if android_packageable_info.prebuilt_native_library_dirs else [] for android_packageable_info in android_packageable_infos]) prebuilt_native_library_targets_for_primary_apk = dedupe([str(native_lib_dir.raw_target) for native_lib_dir in prebuilt_native_library_dirs if native_lib_dir.for_primary_apk]) - if application_module_blocklist or used_by_wrap_script_libs or prebuilt_native_library_targets_for_primary_apk: - all_blocklisted_deps = used_by_wrap_script_libs + prebuilt_native_library_targets_for_primary_apk + if application_module_blocklist or used_by_wrap_script_libs or prebuilt_native_library_targets_for_primary_apk or primary_apk_deps.size() > 0: + all_blocklisted_deps = used_by_wrap_script_libs + prebuilt_native_library_targets_for_primary_apk + primary_apk_deps.list() if application_module_blocklist: all_blocklisted_deps.extend([str(blocklisted_dep.label.raw_target()) for blocklisted_dep in flatten(application_module_blocklist)]) diff --git a/prelude/apple/apple_binary.bzl b/prelude/apple/apple_binary.bzl index 514ec2c23aa42..ded300620bac8 100644 --- a/prelude/apple/apple_binary.bzl +++ b/prelude/apple/apple_binary.bzl @@ -6,7 +6,6 @@ # of this source tree. load("@prelude//:paths.bzl", "paths") -load("@prelude//apple:apple_buck2_compatibility.bzl", "apple_check_buck2_compatibility") load("@prelude//apple:apple_stripping.bzl", "apple_strip_args") # @oss-disable: load("@prelude//apple/meta_only:linker_outputs.bzl", "add_extra_linker_outputs") load( @@ -52,6 +51,7 @@ load( ) load( "@prelude//linking:link_info.bzl", + "CxxSanitizerRuntimeInfo", "LinkCommandDebugOutputInfo", "UnstrippedLinkOutputInfo", ) @@ -63,16 +63,14 @@ load(":apple_code_signing_types.bzl", "AppleEntitlementsInfo") load(":apple_dsym.bzl", "DSYM_SUBTARGET", "get_apple_dsym") load(":apple_entitlements.bzl", "entitlements_link_flags") load(":apple_frameworks.bzl", "get_framework_search_path_flags") -load(":apple_genrule_deps.bzl", "get_apple_build_genrule_deps_attr_value", "get_apple_genrule_deps_outputs") load(":apple_target_sdk_version.bzl", "get_min_deployment_version_for_node", "get_min_deployment_version_target_linker_flags", "get_min_deployment_version_target_preprocessor_flags") load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback") +load(":apple_validation_deps.bzl", "get_apple_validation_deps_outputs") load(":debug.bzl", "AppleDebuggableInfo") load(":resource_groups.bzl", "create_resource_graph") load(":xcode.bzl", "apple_populate_xcode_attributes") def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: - apple_check_buck2_compatibility(ctx) - def get_apple_binary_providers(deps_providers) -> list[Provider]: # FIXME: Ideally we'd like to remove the support of "bridging header", # cause it affects build time and in general considered a bad practise. @@ -82,7 +80,7 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: cxx_srcs, swift_srcs = _filter_swift_srcs(ctx) framework_search_path_flags = get_framework_search_path_flags(ctx) - swift_compile = compile_swift( + swift_compile, _ = compile_swift( ctx, swift_srcs, False, # parse_as_library @@ -111,16 +109,13 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: swift_compile, ) - genrule_deps_outputs = [] - if get_apple_build_genrule_deps_attr_value(ctx): - genrule_deps_outputs = get_apple_genrule_deps_outputs(cxx_attr_deps(ctx)) - + validation_deps_outputs = get_apple_validation_deps_outputs(ctx) stripped = get_apple_stripped_attr_value_with_default_fallback(ctx) constructor_params = CxxRuleConstructorParams( rule_type = "apple_binary", headers_layout = get_apple_cxx_headers_layout(ctx), extra_link_flags = extra_link_flags, - extra_hidden = genrule_deps_outputs, + extra_hidden = validation_deps_outputs, srcs = cxx_srcs, additional = CxxRuleAdditionalParams( srcs = swift_srcs, @@ -190,6 +185,10 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: if cxx_output.link_command_debug_output: link_command_providers.append(LinkCommandDebugOutputInfo(debug_outputs = [cxx_output.link_command_debug_output])) + sanitizer_runtime_providers = [] + if cxx_output.sanitizer_runtime_dir: + sanitizer_runtime_providers.append(CxxSanitizerRuntimeInfo(runtime_dir = cxx_output.sanitizer_runtime_dir)) + return [ DefaultInfo(default_output = cxx_output.binary, sub_targets = cxx_output.sub_targets), RunInfo(args = cmd_args(cxx_output.binary).hidden(cxx_output.runtime_files)), @@ -199,7 +198,7 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: cxx_output.compilation_db, merge_bundle_linker_maps_info(bundle_infos), UnstrippedLinkOutputInfo(artifact = unstripped_binary), - ] + [resource_graph] + min_version_providers + link_command_providers + ] + [resource_graph] + min_version_providers + link_command_providers + sanitizer_runtime_providers if uses_explicit_modules(ctx): return get_swift_anonymous_targets(ctx, get_apple_binary_providers) diff --git a/prelude/apple/apple_bundle.bzl b/prelude/apple/apple_bundle.bzl index c517763edd86c..357e8c1f0f155 100644 --- a/prelude/apple/apple_bundle.bzl +++ b/prelude/apple/apple_bundle.bzl @@ -12,7 +12,6 @@ load( "project_artifacts", ) load("@prelude//:paths.bzl", "paths") -load("@prelude//apple:apple_buck2_compatibility.bzl", "apple_check_buck2_compatibility") load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo", "AppleToolsInfo") # @oss-disable: load("@prelude//apple/meta_only:linker_outputs.bzl", "subtargets_for_apple_bundle_extra_outputs") load("@prelude//apple/user:apple_selected_debug_path_file.bzl", "SELECTED_DEBUG_PATH_FILE_NAME") @@ -61,9 +60,9 @@ load( ) load(":apple_bundle_utility.bzl", "get_bundle_min_target_version", "get_default_binary_dep", "get_flattened_binary_deps", "get_product_name") load(":apple_dsym.bzl", "DSYM_INFO_SUBTARGET", "DSYM_SUBTARGET", "get_apple_dsym", "get_apple_dsym_ext", "get_apple_dsym_info") -load(":apple_genrule_deps.bzl", "get_apple_build_genrule_deps_attr_value", "get_apple_genrule_deps_outputs") load(":apple_sdk.bzl", "get_apple_sdk_name") load(":apple_universal_binaries.bzl", "create_universal_binary") +load(":apple_validation_deps.bzl", "get_apple_validation_deps_outputs") load( ":debug.bzl", "AggregatedAppleDebugInfo", @@ -306,7 +305,6 @@ def _infer_apple_bundle_type(ctx: AnalysisContext) -> AppleBundleType: def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: _apple_bundle_run_validity_checks(ctx) - apple_check_buck2_compatibility(ctx) binary_outputs = _get_binary(ctx) @@ -320,17 +318,14 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: primary_binary_rel_path = get_apple_bundle_part_relative_destination_path(ctx, primary_binary_part) - genrule_deps_outputs = [] - if get_apple_build_genrule_deps_attr_value(ctx): - genrule_deps_outputs = get_apple_genrule_deps_outputs(ctx.attrs.deps) - + validation_deps_outputs = get_apple_validation_deps_outputs(ctx) sub_targets = assemble_bundle( ctx, bundle, apple_bundle_part_list_output.parts, apple_bundle_part_list_output.info_plist_part, SwiftStdlibArguments(primary_binary_rel_path = primary_binary_rel_path), - genrule_deps_outputs, + validation_deps_outputs, ) sub_targets.update(aggregated_debug_info.sub_targets) @@ -465,6 +460,7 @@ def generate_install_data( data = { "fullyQualifiedName": ctx.label, "info_plist": plist_path, + "platform_name": get_apple_sdk_name(ctx), "use_idb": "true", ## TODO(T110665037): read from .buckconfig # We require the user to have run `xcode-select` and `/var/db/xcode_select_link` to symlink diff --git a/prelude/apple/apple_bundle_config.bzl b/prelude/apple/apple_bundle_config.bzl index 507d9f9de0313..7dd9d79bdec51 100644 --- a/prelude/apple/apple_bundle_config.bzl +++ b/prelude/apple/apple_bundle_config.bzl @@ -11,23 +11,11 @@ def _maybe_get_bool(config: str, default: [None, bool]) -> [None, bool]: return default return result.lower() == "true" -def _get_bundling_path_conflicts_check_enabled(): - check_enabled = _maybe_get_bool("bundling_path_conflicts_check_enabled", None) - if check_enabled != None: - return check_enabled - - return select({ - "DEFAULT": True, - "ovr_config//features/apple/constraints:bundling_path_conflicts_check_disabled": False, - "ovr_config//features/apple/constraints:bundling_path_conflicts_check_enabled": True, - }) - def apple_bundle_config() -> dict[str, typing.Any]: return { "_bundling_cache_buster": read_root_config("apple", "bundling_cache_buster", None), "_bundling_log_file_enabled": _maybe_get_bool("bundling_log_file_enabled", True), "_bundling_log_file_level": read_root_config("apple", "bundling_log_file_level", None), - "_bundling_path_conflicts_check_enabled": _get_bundling_path_conflicts_check_enabled(), "_codesign_type": read_root_config("apple", "codesign_type_override", None), "_compile_resources_locally_override": _maybe_get_bool("compile_resources_locally_override", None), "_dry_run_code_signing": _maybe_get_bool("dry_run_code_signing", False), diff --git a/prelude/apple/apple_bundle_part.bzl b/prelude/apple/apple_bundle_part.bzl index b51c73ab51577..8f53e9da593c1 100644 --- a/prelude/apple/apple_bundle_part.bzl +++ b/prelude/apple/apple_bundle_part.bzl @@ -177,9 +177,7 @@ def assemble_bundle( command.add("--log-level-file", ctx.attrs._bundling_log_file_level) subtargets["bundling-log"] = [DefaultInfo(default_output = bundling_log_output)] - if ctx.attrs._bundling_path_conflicts_check_enabled: - command.add("--check-conflicts") - + command.add("--check-conflicts") command.add(codesign_configuration_args) # Ensures any genrule deps get built, such targets are used for validation diff --git a/prelude/apple/apple_bundle_resources.bzl b/prelude/apple/apple_bundle_resources.bzl index 0dbc6e93f88c6..a8d07747e8375 100644 --- a/prelude/apple/apple_bundle_resources.bzl +++ b/prelude/apple/apple_bundle_resources.bzl @@ -8,6 +8,10 @@ load("@prelude//:artifacts.bzl", "single_artifact") load("@prelude//:paths.bzl", "paths") load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo") +load( + "@prelude//linking:link_info.bzl", + "CxxSanitizerRuntimeInfo", +) load("@prelude//utils:utils.bzl", "flatten_dict") load( ":apple_asset_catalog.bzl", @@ -58,6 +62,7 @@ def get_apple_bundle_resource_part_list(ctx: AnalysisContext) -> AppleBundleReso parts = [] parts.extend(_create_pkg_info_if_needed(ctx)) + parts.extend(_copy_privacy_manifest_if_needed(ctx)) (resource_specs, asset_catalog_specs, core_data_specs, scene_kit_assets_spec, cxx_resource_specs) = _select_resources(ctx) @@ -79,6 +84,15 @@ def get_apple_bundle_resource_part_list(ctx: AnalysisContext) -> AppleBundleReso ), ) + cxx_sanitizer_runtime_info = ctx.attrs.binary.get(CxxSanitizerRuntimeInfo) if ctx.attrs.binary else None + if cxx_sanitizer_runtime_info: + runtime_resource_spec = AppleResourceSpec( + content_dirs = [cxx_sanitizer_runtime_info.runtime_dir], + destination = AppleResourceDestination("frameworks"), + codesign_files_on_copy = True, + ) + resource_specs.append(runtime_resource_spec) + asset_catalog_result = compile_apple_asset_catalog(ctx, asset_catalog_specs) if asset_catalog_result != None: asset_catalog_part = AppleBundlePart( @@ -128,6 +142,19 @@ def _create_pkg_info_if_needed(ctx: AnalysisContext) -> list[AppleBundlePart]: artifact = ctx.actions.write("PkgInfo", "APPLWRUN\n") return [AppleBundlePart(source = artifact, destination = AppleBundleDestination("metadata"))] +def _copy_privacy_manifest_if_needed(ctx: AnalysisContext) -> list[AppleBundlePart]: + privacy_manifest = ctx.attrs.privacy_manifest + if privacy_manifest == None: + return [] + + # According to apple docs, privacy manifest has to be named as `PrivacyInfo.xcprivacy` + if privacy_manifest.short_path.split("/", 1)[-1] == "PrivacyInfo.xcprivacy": + artifact = privacy_manifest + else: + output = ctx.actions.declare_output("PrivacyInfo.xcprivacy") + artifact = ctx.actions.copy_file(output.as_output(), privacy_manifest) + return [AppleBundlePart(source = artifact, destination = AppleBundleDestination("metadata"))] + def _select_resources(ctx: AnalysisContext) -> ((list[AppleResourceSpec], list[AppleAssetCatalogSpec], list[AppleCoreDataSpec], list[SceneKitAssetsSpec], list[CxxResourceSpec])): resource_group_info = get_resource_group_info(ctx) if resource_group_info: diff --git a/prelude/apple/apple_framework_versions.bzl b/prelude/apple/apple_framework_versions.bzl index 3f6761b8a2068..9b86b910f41ed 100644 --- a/prelude/apple/apple_framework_versions.bzl +++ b/prelude/apple/apple_framework_versions.bzl @@ -660,6 +660,7 @@ _FRAMEWORK_INTRODUCED_VERSIONS = { "MobileCoreServices": { "appletvos": (9, 0, 0), "iphoneos": (2, 0, 0), + "maccatalyst": (14, 0, 0), "watchos": (1, 0, 0), }, "ModelIO": { diff --git a/prelude/apple/apple_genrule_deps.bzl b/prelude/apple/apple_genrule_deps.bzl deleted file mode 100644 index 756a5bdd3ed6b..0000000000000 --- a/prelude/apple/apple_genrule_deps.bzl +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under both the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree and the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. - -load("@prelude//:genrule_types.bzl", "GENRULE_MARKER_SUBTARGET_NAME") - -def get_apple_genrule_deps_outputs(deps: list[Dependency]) -> list[Artifact]: - artifacts = [] - for dep in deps: - default_info = dep[DefaultInfo] - if GENRULE_MARKER_SUBTARGET_NAME in default_info.sub_targets: - artifacts += default_info.default_outputs - return artifacts - -def get_apple_build_genrule_deps_attr_value(ctx: AnalysisContext) -> bool: - build_genrule_deps = ctx.attrs.build_genrule_deps - if build_genrule_deps != None: - # `build_genrule_deps` present on a target takes priority - return build_genrule_deps - - # Fallback to the default value which is driven by buckconfig + select() - return ctx.attrs._build_genrule_deps - -def get_apple_build_genrule_deps_default_kwargs() -> dict[str, typing.Any]: - return { - APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME: _build_genrule_deps_default_enabled(), - } - -def _build_genrule_deps_default_enabled() -> typing.Any: - buckconfig_value = read_root_config("apple", "build_genrule_deps", None) - if buckconfig_value != None: - return buckconfig_value.lower() == "true" - - return select({ - "DEFAULT": False, - # TODO(mgd): Make `config//` references possible from macro layer - "ovr_config//features/apple/constraints:build_genrule_deps_enabled": True, - }) - -APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME = "_build_genrule_deps" -APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE = attrs.bool(default = False) - -APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME = "build_genrule_deps" -APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE = attrs.option(attrs.bool(), default = None) diff --git a/prelude/apple/apple_library.bzl b/prelude/apple/apple_library.bzl index ce7cf47040daf..afe9e6f3418e1 100644 --- a/prelude/apple/apple_library.bzl +++ b/prelude/apple/apple_library.bzl @@ -9,7 +9,6 @@ load( "@prelude//:artifact_tset.bzl", "project_artifacts", ) -load("@prelude//apple:apple_buck2_compatibility.bzl", "apple_check_buck2_compatibility") load("@prelude//apple:apple_dsym.bzl", "DSYM_SUBTARGET", "get_apple_dsym") load("@prelude//apple:apple_stripping.bzl", "apple_strip_args") load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo") @@ -71,10 +70,10 @@ load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//utils:expect.bzl", "expect") load(":apple_bundle_types.bzl", "AppleBundleLinkerMapInfo", "AppleMinDeploymentVersionInfo") load(":apple_frameworks.bzl", "get_framework_search_path_flags") -load(":apple_genrule_deps.bzl", "get_apple_build_genrule_deps_attr_value", "get_apple_genrule_deps_outputs") load(":apple_modular_utility.bzl", "MODULE_CACHE_PATH") load(":apple_target_sdk_version.bzl", "get_min_deployment_version_for_node", "get_min_deployment_version_target_linker_flags", "get_min_deployment_version_target_preprocessor_flags") load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback", "get_module_name") +load(":apple_validation_deps.bzl", "get_apple_validation_deps_outputs") load( ":debug.bzl", "AppleDebuggableInfo", @@ -108,8 +107,6 @@ AppleLibraryAdditionalParams = record( ) def apple_library_impl(ctx: AnalysisContext) -> [Promise, list[Provider]]: - apple_check_buck2_compatibility(ctx) - def get_apple_library_providers(deps_providers) -> list[Provider]: constructor_params = apple_library_rule_constructor_params_and_swift_providers( ctx, @@ -145,7 +142,7 @@ def apple_library_rule_constructor_params_and_swift_providers(ctx: AnalysisConte modulemap_pre = None framework_search_paths_flags = get_framework_search_path_flags(ctx) - swift_compile = compile_swift( + swift_compile, swift_interface = compile_swift( ctx, swift_srcs, True, # parse_as_library @@ -221,16 +218,13 @@ def apple_library_rule_constructor_params_and_swift_providers(ctx: AnalysisConte relative_args = CPreprocessorArgs(args = [framework_search_paths_flags]), ) - genrule_deps_outputs = [] - if get_apple_build_genrule_deps_attr_value(ctx): - genrule_deps_outputs = get_apple_genrule_deps_outputs(cxx_attr_deps(ctx) + cxx_attr_exported_deps(ctx)) - + validation_deps_outputs = get_apple_validation_deps_outputs(ctx) return CxxRuleConstructorParams( rule_type = params.rule_type, is_test = (params.rule_type == "apple_test"), headers_layout = get_apple_cxx_headers_layout(ctx), extra_exported_link_flags = params.extra_exported_link_flags, - extra_hidden = genrule_deps_outputs, + extra_hidden = validation_deps_outputs, extra_link_flags = [_get_linker_flags(ctx)], extra_link_input = swift_object_files, extra_link_input_has_external_debug_info = True, @@ -257,6 +251,7 @@ def apple_library_rule_constructor_params_and_swift_providers(ctx: AnalysisConte default_outputs = swift_compile.object_files if swift_compile else None, ), ], + "swift-interface": [swift_interface], "swift-output-file-map": [ DefaultInfo( default_output = swift_compile.output_map_artifact if swift_compile else None, diff --git a/prelude/apple/apple_macro_layer.bzl b/prelude/apple/apple_macro_layer.bzl index faf2fab3dd261..3d39921e1e835 100644 --- a/prelude/apple/apple_macro_layer.bzl +++ b/prelude/apple/apple_macro_layer.bzl @@ -7,7 +7,6 @@ load(":apple_bundle_config.bzl", "apple_bundle_config") load(":apple_dsym_config.bzl", "apple_dsym_config") -load(":apple_genrule_deps.bzl", "get_apple_build_genrule_deps_default_kwargs") load(":apple_info_plist_substitutions_parsing.bzl", "parse_codesign_entitlements") load(":apple_package_config.bzl", "apple_package_config") load(":apple_resource_bundle.bzl", "make_resource_bundle_rule") @@ -76,7 +75,6 @@ def apple_test_macro_impl(apple_test_rule, apple_resource_bundle_rule, **kwargs) kwargs.update(apple_bundle_config()) kwargs.update(apple_dsym_config()) kwargs.update(apple_macro_layer_set_bool_override_attrs_from_config(_APPLE_TEST_LOCAL_EXECUTION_OVERRIDES)) - kwargs.update(get_apple_build_genrule_deps_default_kwargs()) # `extension` is used both by `apple_test` and `apple_resource_bundle`, so provide default here kwargs["extension"] = kwargs.pop("extension", "xctest") @@ -89,7 +87,6 @@ def apple_bundle_macro_impl(apple_bundle_rule, apple_resource_bundle_rule, **kwa info_plist_substitutions = kwargs.get("info_plist_substitutions") kwargs.update(apple_bundle_config()) kwargs.update(apple_dsym_config()) - kwargs.update(get_apple_build_genrule_deps_default_kwargs()) apple_bundle_rule( _codesign_entitlements = parse_codesign_entitlements(info_plist_substitutions), _resource_bundle = make_resource_bundle_rule(apple_resource_bundle_rule, **kwargs), @@ -100,7 +97,6 @@ def apple_library_macro_impl(apple_library_rule = None, **kwargs): kwargs.update(apple_dsym_config()) kwargs.update(apple_macro_layer_set_bool_override_attrs_from_config(_APPLE_LIBRARY_LOCAL_EXECUTION_OVERRIDES)) kwargs.update(apple_macro_layer_set_bool_override_attrs_from_config([APPLE_STRIPPED_DEFAULT])) - kwargs.update(get_apple_build_genrule_deps_default_kwargs()) apple_library_rule(**kwargs) def apple_binary_macro_impl(apple_binary_rule = None, apple_universal_executable = None, **kwargs): @@ -108,7 +104,6 @@ def apple_binary_macro_impl(apple_binary_rule = None, apple_universal_executable kwargs.update(dsym_args) kwargs.update(apple_macro_layer_set_bool_override_attrs_from_config(_APPLE_BINARY_LOCAL_EXECUTION_OVERRIDES)) kwargs.update(apple_macro_layer_set_bool_override_attrs_from_config([APPLE_STRIPPED_DEFAULT])) - kwargs.update(get_apple_build_genrule_deps_default_kwargs()) original_binary_name = kwargs.pop("name") diff --git a/prelude/apple/apple_package.bzl b/prelude/apple/apple_package.bzl index fc0227933f1f6..bc35f7ffb8781 100644 --- a/prelude/apple/apple_package.bzl +++ b/prelude/apple/apple_package.bzl @@ -17,34 +17,43 @@ load(":apple_toolchain_types.bzl", "AppleToolchainInfo", "AppleToolsInfo") def apple_package_impl(ctx: AnalysisContext) -> list[Provider]: package = ctx.actions.declare_output("{}.{}".format(ctx.attrs.bundle.label.name, ctx.attrs.ext)) + contents = ( + ctx.attrs.bundle[DefaultInfo].default_outputs[0] if ctx.attrs.packager else _get_ipa_contents(ctx) + ) if ctx.attrs.packager: process_ipa_cmd = cmd_args([ ctx.attrs.packager[RunInfo], "--app-bundle-path", - ctx.attrs.bundle[DefaultInfo].default_outputs[0], + contents, "--output-path", package.as_output(), ctx.attrs.packager_args, ]) category = "apple_package_make_custom" else: - unprocessed_ipa_contents = _get_ipa_contents(ctx) process_ipa_cmd = _get_default_package_cmd( ctx, - unprocessed_ipa_contents, + contents, package.as_output(), ) category = "apple_package_make" - if ctx.attrs.validator != None: - process_ipa_cmd.add([ - "--validator", - ctx.attrs.validator[RunInfo], - [cmd_args(["--validator-args=", arg], delimiter = "") for arg in ctx.attrs.validator_args], - ]) + sub_targets = {} + + prepackaged_validators_artifacts = _get_prepackaged_validators_outputs(ctx, contents) + if prepackaged_validators_artifacts: + # Add the artifacts to packaging cmd so that they are run. + process_ipa_cmd.hidden(prepackaged_validators_artifacts) + sub_targets["prepackaged_validators"] = [ + DefaultInfo(default_outputs = prepackaged_validators_artifacts), + ] + ctx.actions.run(process_ipa_cmd, category = category) - return [DefaultInfo(default_output = package)] + return [DefaultInfo( + default_output = package, + sub_targets = sub_targets, + )] def _get_default_package_cmd(ctx: AnalysisContext, unprocessed_ipa_contents: Artifact, output: OutputArtifact) -> cmd_args: apple_tools = ctx.attrs._apple_tools[AppleToolsInfo] @@ -161,3 +170,33 @@ def _compression_level_arg(compression_level: IpaCompressionLevel) -> str: return "9" else: fail("Unknown .ipa compression level: " + str(compression_level)) + +def _get_prepackaged_validators_outputs(ctx: AnalysisContext, prepackaged_contents: Artifact) -> list[Artifact]: + if not ctx.attrs.prepackaged_validators: + return [] + + outputs = [] + for idx, validator in enumerate(ctx.attrs.prepackaged_validators): + if type(validator) == "tuple": + validator, validator_args = validator + else: + validator = validator + validator_args = [] + + output = ctx.actions.declare_output(validator.label.name + "_{}".format(idx)) + outputs.append(output) + + ctx.actions.run( + cmd_args([ + validator[RunInfo], + "--contents-dir", + prepackaged_contents, + "--output-path", + output.as_output(), + validator_args, + ]), + category = "prepackaged_validator", + identifier = str(idx), + ) + + return outputs diff --git a/prelude/apple/apple_resource_bundle.bzl b/prelude/apple/apple_resource_bundle.bzl index 0ed45dfb4d80a..2ea6fae294705 100644 --- a/prelude/apple/apple_resource_bundle.bzl +++ b/prelude/apple/apple_resource_bundle.bzl @@ -51,6 +51,7 @@ _RESOURCE_BUNDLE_FIELDS = [ "info_plist", "info_plist_substitutions", "product_name", + "privacy_manifest", "resource_group", "resource_group_map", "within_view", diff --git a/prelude/apple/apple_rules_impl.bzl b/prelude/apple/apple_rules_impl.bzl index 3674ed732c9b9..a62e4bc6257c1 100644 --- a/prelude/apple/apple_rules_impl.bzl +++ b/prelude/apple/apple_rules_impl.bzl @@ -5,14 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//apple:apple_buck2_compatibility.bzl", "BUCK2_COMPATIBILITY_ATTRIB_NAME", "BUCK2_COMPATIBILITY_ATTRIB_TYPE") -load( - "@prelude//apple:apple_genrule_deps.bzl", - "APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME", - "APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE", - "APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME", - "APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE", -) +load("@prelude//:buck2_compatibility.bzl", "BUCK2_COMPATIBILITY_ATTRIB_NAME", "BUCK2_COMPATIBILITY_ATTRIB_TYPE") load("@prelude//apple/swift:swift_incremental_support.bzl", "SwiftCompilationMode") load("@prelude//apple/swift:swift_toolchain.bzl", "swift_toolchain_impl") load("@prelude//apple/swift:swift_toolchain_types.bzl", "SwiftObjectFormat") @@ -34,9 +27,12 @@ load(":apple_resource.bzl", "apple_resource_impl") load( ":apple_rules_impl_utility.bzl", "APPLE_ARCHIVE_OBJECTS_LOCALLY_OVERRIDE_ATTR_NAME", + "APPLE_VALIDATION_DEPS_ATTR_NAME", + "APPLE_VALIDATION_DEPS_ATTR_TYPE", "apple_bundle_extra_attrs", "apple_dsymutil_attrs", "apple_test_extra_attrs", + "apple_xcuitest_extra_attrs", "get_apple_bundle_toolchain_attr", "get_apple_toolchain_attr", "get_apple_xctoolchain_attr", @@ -46,6 +42,7 @@ load(":apple_test.bzl", "apple_test_impl") load(":apple_toolchain.bzl", "apple_toolchain_impl") load(":apple_toolchain_types.bzl", "AppleToolsInfo") load(":apple_universal_executable.bzl", "apple_universal_executable_impl") +load(":apple_xcuitest.bzl", "apple_xcuitest_impl") load(":prebuilt_apple_framework.bzl", "prebuilt_apple_framework_impl") load(":scene_kit_assets.bzl", "scene_kit_assets_impl") load(":xcode_postbuild_script.bzl", "xcode_postbuild_script_impl") @@ -61,6 +58,7 @@ implemented_rules = { "apple_test": apple_test_impl, "apple_toolchain": apple_toolchain_impl, "apple_universal_executable": apple_universal_executable_impl, + "apple_xcuitest": apple_xcuitest_impl, "core_data_model": apple_core_data_impl, "prebuilt_apple_framework": prebuilt_apple_framework_impl, "scene_kit_assets": scene_kit_assets_impl, @@ -89,6 +87,7 @@ def _apple_binary_extra_attrs(): "precompiled_header": attrs.option(attrs.dep(providers = [CPrecompiledHeaderInfo]), default = None), "prefer_stripped_objects": attrs.bool(default = False), "preferred_linkage": attrs.enum(Linkage, default = "any"), + "sanitizer_runtime_enabled": attrs.option(attrs.bool(), default = None), "stripped": attrs.option(attrs.bool(), default = None), "swift_compilation_mode": attrs.enum(SwiftCompilationMode.values(), default = "wmo"), "_apple_toolchain": _APPLE_TOOLCHAIN_ATTR, @@ -96,9 +95,8 @@ def _apple_binary_extra_attrs(): "_apple_xctoolchain": get_apple_xctoolchain_attr(), "_apple_xctoolchain_bundle_id": get_apple_xctoolchain_bundle_id_attr(), "_stripped_default": attrs.bool(default = False), - APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE, - APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE, BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, + APPLE_VALIDATION_DEPS_ATTR_NAME: APPLE_VALIDATION_DEPS_ATTR_TYPE, } attribs.update(apple_dsymutil_attrs()) return attribs @@ -124,9 +122,8 @@ def _apple_library_extra_attrs(): "_apple_xctoolchain_bundle_id": get_apple_xctoolchain_bundle_id_attr(), "_stripped_default": attrs.bool(default = False), APPLE_ARCHIVE_OBJECTS_LOCALLY_OVERRIDE_ATTR_NAME: attrs.option(attrs.bool(), default = None), - APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE, - APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE, BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, + APPLE_VALIDATION_DEPS_ATTR_NAME: APPLE_VALIDATION_DEPS_ATTR_TYPE, } attribs.update(apple_dsymutil_attrs()) return attribs @@ -156,8 +153,13 @@ extra_attributes = { "ext": attrs.enum(ApplePackageExtension.values(), default = "ipa"), "packager": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "packager_args": attrs.list(attrs.arg(), default = []), - "validator": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), - "validator_args": attrs.list(attrs.arg(), default = []), + "prepackaged_validators": attrs.list( + attrs.one_of( + attrs.exec_dep(providers = [RunInfo]), + attrs.tuple(attrs.exec_dep(providers = [RunInfo]), attrs.list(attrs.arg())), + ), + default = [], + ), "_apple_toolchain": get_apple_bundle_toolchain_attr(), "_apple_tools": attrs.exec_dep(default = "prelude//apple/tools:apple-tools", providers = [AppleToolsInfo]), "_ipa_compression_level": attrs.enum(IpaCompressionLevel.values()), @@ -191,6 +193,7 @@ extra_attributes = { "lipo": attrs.exec_dep(providers = [RunInfo]), "min_version": attrs.option(attrs.string(), default = None), "momc": attrs.exec_dep(providers = [RunInfo]), + "objdump": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "odrcov": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), # A placeholder tool that can be used to set up toolchain constraints. # Useful when fat and thin toolchahins share the same underlying tools via `command_alias()`, @@ -215,6 +218,7 @@ extra_attributes = { "_internal_sdk_path": attrs.option(attrs.string(), default = None), }, "apple_universal_executable": _apple_universal_executable_extra_attrs(), + "apple_xcuitest": apple_xcuitest_extra_attrs(), "core_data_model": { "path": attrs.source(allow_directory = True), }, @@ -232,6 +236,7 @@ extra_attributes = { "swift_toolchain": { "architecture": attrs.option(attrs.string(), default = None), # TODO(T115173356): Make field non-optional "make_swift_comp_db": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//apple/tools:make_swift_comp_db")), + "make_swift_interface": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//apple/tools:make_swift_interface")), "object_format": attrs.enum(SwiftObjectFormat.values(), default = "object"), # A placeholder tool that can be used to set up toolchain constraints. # Useful when fat and thin toolchahins share the same underlying tools via `command_alias()`, @@ -240,6 +245,7 @@ extra_attributes = { "platform_path": attrs.option(attrs.source(), default = None), # Mark as optional until we remove `_internal_platform_path` "sdk_modules": attrs.list(attrs.exec_dep(), default = []), # A list or a root target that represent a graph of sdk modules (e.g Frameworks) "sdk_path": attrs.option(attrs.source(), default = None), # Mark as optional until we remove `_internal_sdk_path` + "swift_ide_test_tool": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "swift_stdlib_tool": attrs.exec_dep(providers = [RunInfo]), "swiftc": attrs.exec_dep(providers = [RunInfo]), # TODO(T111858757): Mirror of `platform_path` but treated as a string. It allows us to diff --git a/prelude/apple/apple_rules_impl_utility.bzl b/prelude/apple/apple_rules_impl_utility.bzl index f8d157ba4906b..63c3d87ad9921 100644 --- a/prelude/apple/apple_rules_impl_utility.bzl +++ b/prelude/apple/apple_rules_impl_utility.bzl @@ -5,17 +5,10 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//apple:apple_buck2_compatibility.bzl", "BUCK2_COMPATIBILITY_ATTRIB_NAME", "BUCK2_COMPATIBILITY_ATTRIB_TYPE") +load("@prelude//:buck2_compatibility.bzl", "BUCK2_COMPATIBILITY_ATTRIB_NAME", "BUCK2_COMPATIBILITY_ATTRIB_TYPE") load("@prelude//apple:apple_bundle_attrs.bzl", "get_apple_info_plist_build_system_identification_attrs") load("@prelude//apple:apple_bundle_types.bzl", "AppleBundleResourceInfo", "AppleBundleTypeAttributeType") load("@prelude//apple:apple_code_signing_types.bzl", "CodeSignType") -load( - "@prelude//apple:apple_genrule_deps.bzl", - "APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME", - "APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE", - "APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME", - "APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE", -) load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo", "AppleToolsInfo") load("@prelude//apple/swift:swift_incremental_support.bzl", "SwiftCompilationMode") load("@prelude//apple/user:apple_selective_debugging.bzl", "AppleSelectiveDebuggingInfo") @@ -47,6 +40,9 @@ APPLE_ARCHIVE_OBJECTS_LOCALLY_OVERRIDE_ATTR_NAME = "_archive_objects_locally_ove APPLE_USE_ENTITLEMENTS_WHEN_ADHOC_CODE_SIGNING_CONFIG_OVERRIDE_ATTR_NAME = "_use_entitlements_when_adhoc_code_signing" APPLE_USE_ENTITLEMENTS_WHEN_ADHOC_CODE_SIGNING_ATTR_NAME = "use_entitlements_when_adhoc_code_signing" +APPLE_VALIDATION_DEPS_ATTR_NAME = "validation_deps" +APPLE_VALIDATION_DEPS_ATTR_TYPE = attrs.set(attrs.dep(), sorted = True, default = []) + def apple_dsymutil_attrs(): return { "_dsymutil_extra_flags": attrs.list(attrs.string()), @@ -62,7 +58,6 @@ def _apple_bundle_like_common_attrs(): "_bundling_cache_buster": attrs.option(attrs.string(), default = None), "_bundling_log_file_enabled": attrs.bool(default = False), "_bundling_log_file_level": attrs.option(attrs.string(), default = None), - "_bundling_path_conflicts_check_enabled": attrs.bool(default = False), "_codesign_type": attrs.option(attrs.enum(CodeSignType.values()), default = None), "_compile_resources_locally_override": attrs.option(attrs.bool(), default = None), "_dry_run_code_signing": attrs.bool(default = False), @@ -75,6 +70,7 @@ def _apple_bundle_like_common_attrs(): APPLE_USE_ENTITLEMENTS_WHEN_ADHOC_CODE_SIGNING_CONFIG_OVERRIDE_ATTR_NAME: attrs.option(attrs.bool(), default = None), APPLE_USE_ENTITLEMENTS_WHEN_ADHOC_CODE_SIGNING_ATTR_NAME: attrs.bool(default = False), BUCK2_COMPATIBILITY_ATTRIB_NAME: BUCK2_COMPATIBILITY_ATTRIB_TYPE, + APPLE_VALIDATION_DEPS_ATTR_NAME: APPLE_VALIDATION_DEPS_ATTR_TYPE, } attribs.update(get_apple_info_plist_build_system_identification_attrs()) attribs.update(apple_dsymutil_attrs()) @@ -104,16 +100,33 @@ def apple_test_extra_attrs(): "resource_group_map": attrs.option(attrs.string(), default = None), "stripped": attrs.bool(default = False), "swift_compilation_mode": attrs.enum(SwiftCompilationMode.values(), default = "wmo"), + "use_m1_simulator": attrs.bool(default = False), "_apple_toolchain": get_apple_toolchain_attr(), "_ios_booted_simulator": attrs.transition_dep(cfg = apple_simulators_transition, default = "fbsource//xplat/buck2/platform/apple:ios_booted_simulator", providers = [LocalResourceInfo]), "_ios_unbooted_simulator": attrs.transition_dep(cfg = apple_simulators_transition, default = "fbsource//xplat/buck2/platform/apple:ios_unbooted_simulator", providers = [LocalResourceInfo]), "_macos_idb_companion": attrs.transition_dep(cfg = apple_simulators_transition, default = "fbsource//xplat/buck2/platform/apple:macos_idb_companion", providers = [LocalResourceInfo]), } attribs.update(_apple_bundle_like_common_attrs()) - attribs.update({ - APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE, - APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE, - }) + return attribs + +def apple_xcuitest_extra_attrs(): + attribs = { + # This is ignored, but required for info plist processing. + "binary": attrs.option(attrs.source(), default = None), + "codesign_identity": attrs.option(attrs.string(), default = None), + "entitlements_file": attrs.option(attrs.source(), default = None), + "extension": attrs.default_only(attrs.string(default = "app")), + "incremental_bundling_enabled": attrs.bool(default = False), + "info_plist": attrs.source(), + "info_plist_substitutions": attrs.dict(key = attrs.string(), value = attrs.string(), sorted = False, default = {}), + "target_sdk_version": attrs.option(attrs.string(), default = None), + # The test bundle to package in the UI test runner app. + "test_bundle": attrs.dep(), + "_apple_toolchain": get_apple_toolchain_attr(), + } + attribs.update(_apple_bundle_like_common_attrs()) + attribs.pop("_dsymutil_extra_flags", None) + return attribs def apple_bundle_extra_attrs(): @@ -126,8 +139,6 @@ def apple_bundle_extra_attrs(): "universal": attrs.option(attrs.bool(), default = None), "_apple_toolchain": get_apple_bundle_toolchain_attr(), "_codesign_entitlements": attrs.option(attrs.source(), default = None), - APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_DEFAULT_ATTRIB_TYPE, - APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_NAME: APPLE_BUILD_GENRULE_DEPS_TARGET_ATTRIB_TYPE, } attribs.update(_apple_bundle_like_common_attrs()) return attribs diff --git a/prelude/apple/apple_test.bzl b/prelude/apple/apple_test.bzl index ea5a765fda069..fd74d3d385d92 100644 --- a/prelude/apple/apple_test.bzl +++ b/prelude/apple/apple_test.bzl @@ -6,7 +6,6 @@ # of this source tree. load("@prelude//:paths.bzl", "paths") -load("@prelude//apple:apple_buck2_compatibility.bzl", "apple_check_buck2_compatibility") load("@prelude//apple:apple_library.bzl", "AppleLibraryAdditionalParams", "apple_library_rule_constructor_params_and_swift_providers") load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo") # @oss-disable: load("@prelude//apple/meta_only:apple_test_re_capabilities.bzl", "ios_test_re_capabilities", "macos_test_re_capabilities") @@ -48,8 +47,6 @@ load(":xcode.bzl", "apple_populate_xcode_attributes") load(":xctest_swift_support.bzl", "XCTestSwiftSupportInfo") def apple_test_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: - apple_check_buck2_compatibility(ctx) - def get_apple_test_providers(deps_providers) -> list[Provider]: xctest_bundle = bundle_output(ctx) @@ -199,10 +196,10 @@ def _get_test_info(ctx: AnalysisContext, xctest_bundle: Artifact, test_host_app_ else: # @oss-disable: requires_ios_booted_simulator = ctx.attrs.test_host_app != None or ctx.attrs.ui_test_target_app != None - # @oss-disable: remote_execution_properties = ios_test_re_capabilities(use_unbooted_simulator = not requires_ios_booted_simulator) + # @oss-disable: remote_execution_properties = ios_test_re_capabilities(use_unbooted_simulator = not requires_ios_booted_simulator, use_m1_simulator = ctx.attrs.use_m1_simulator) remote_execution_properties = None # @oss-enable - # @oss-disable: remote_execution_use_case = apple_test_re_use_case(macos_test = sdk_name == MacOSXSdkMetadata.name) + # @oss-disable: remote_execution_use_case = apple_test_re_use_case(macos_test = sdk_name == MacOSXSdkMetadata.name, use_m1_simulator = ctx.attrs.use_m1_simulator) remote_execution_use_case = None # @oss-enable local_enabled = remote_execution_use_case == None diff --git a/prelude/apple/apple_toolchain.bzl b/prelude/apple/apple_toolchain.bzl index 2f84d2506ce9c..e13b18df3146f 100644 --- a/prelude/apple/apple_toolchain.bzl +++ b/prelude/apple/apple_toolchain.bzl @@ -32,6 +32,7 @@ def apple_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: lipo = ctx.attrs.lipo[RunInfo], min_version = ctx.attrs.min_version, momc = ctx.attrs.momc[RunInfo], + objdump = ctx.attrs.objdump[RunInfo] if ctx.attrs.objdump else None, odrcov = ctx.attrs.odrcov[RunInfo] if ctx.attrs.odrcov else None, platform_path = platform_path, sdk_build_version = ctx.attrs.build_version, diff --git a/prelude/apple/apple_toolchain_types.bzl b/prelude/apple/apple_toolchain_types.bzl index 0f7435a91ab47..96adbaa9aff9c 100644 --- a/prelude/apple/apple_toolchain_types.bzl +++ b/prelude/apple/apple_toolchain_types.bzl @@ -25,6 +25,7 @@ AppleToolchainInfo = provider( "lipo": provider_field(typing.Any, default = None), # "RunInfo" "min_version": provider_field(typing.Any, default = None), # [None, str] "momc": provider_field(typing.Any, default = None), # "RunInfo" + "objdump": provider_field(RunInfo | None, default = None), "odrcov": provider_field(typing.Any, default = None), # ["RunInfo", None] "platform_path": provider_field(typing.Any, default = None), # [str, artifact] "sdk_build_version": provider_field(typing.Any, default = None), # "[None, str]" diff --git a/prelude/apple/apple_universal_binaries.bzl b/prelude/apple/apple_universal_binaries.bzl index 43c76fc80142c..f5c3fbe6ee8f1 100644 --- a/prelude/apple/apple_universal_binaries.bzl +++ b/prelude/apple/apple_universal_binaries.bzl @@ -26,8 +26,13 @@ def create_universal_binary( lipo_cmd.add(["-create", "-output", binary_output.as_output()]) ctx.actions.run(lipo_cmd, category = "lipo") + # Universal binaries can be created out of plain `cxx_binary()` / `cxx_library()` + # which lack the `AppleDebuggableInfo` provider. + # TODO(T174234334): Uniformly support debuggable info for apple_*/cxx_* + contains_full_debuggable_info = _all_binaries_have_apple_debuggable_info(binary_deps) + dsym_output = None - if split_arch_dsym: + if split_arch_dsym and contains_full_debuggable_info: dsym_output = ctx.actions.declare_output("UniversalBinary.dSYM" if dsym_bundle_name == None else dsym_bundle_name, dir = True) dsym_combine_cmd = cmd_args([ctx.attrs._apple_tools[AppleToolsInfo].split_arch_combine_dsym_bundles_tool]) @@ -36,7 +41,9 @@ def create_universal_binary( dsym_combine_cmd.add(["--output", dsym_output.as_output()]) ctx.actions.run(dsym_combine_cmd, category = "universal_binaries_dsym") - all_debug_info_tsets = [binary.get(AppleDebuggableInfo).debug_info_tset for binary in binary_deps.values()] + all_debug_info_tsets = [] + if contains_full_debuggable_info: + all_debug_info_tsets = [binary.get(AppleDebuggableInfo).debug_info_tset for binary in binary_deps.values()] return AppleBundleBinaryOutput( binary = binary_output, @@ -50,3 +57,10 @@ def create_universal_binary( ), ), ) + +def _all_binaries_have_apple_debuggable_info(binary_deps: dict[str, Dependency]) -> bool: + for binary in binary_deps.values(): + info = binary.get(AppleDebuggableInfo) + if info == None: + return False + return True diff --git a/prelude/apple/apple_validation_deps.bzl b/prelude/apple/apple_validation_deps.bzl new file mode 100644 index 0000000000000..fa948a4f6ac11 --- /dev/null +++ b/prelude/apple/apple_validation_deps.bzl @@ -0,0 +1,17 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +_VALIDATION_DEPS_ATTR_NAME = "validation_deps" + +def get_apple_validation_deps_outputs(ctx: AnalysisContext) -> list[Artifact]: + artifacts = [] + if hasattr(ctx.attrs, _VALIDATION_DEPS_ATTR_NAME): + validation_deps = getattr(ctx.attrs, _VALIDATION_DEPS_ATTR_NAME) + for dep in validation_deps: + default_info = dep[DefaultInfo] + artifacts += default_info.default_outputs + return artifacts diff --git a/prelude/apple/apple_xcuitest.bzl b/prelude/apple/apple_xcuitest.bzl new file mode 100644 index 0000000000000..14941bfd58561 --- /dev/null +++ b/prelude/apple/apple_xcuitest.bzl @@ -0,0 +1,70 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@prelude//:paths.bzl", "paths") +load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo") +load(":apple_bundle_destination.bzl", "AppleBundleDestination") +load(":apple_bundle_part.bzl", "AppleBundlePart", "assemble_bundle") +load(":apple_info_plist.bzl", "process_info_plist") + +def apple_xcuitest_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: + # The XCUITest runner app bundle copies the application from the platform + # directory, and includes the UI test bundle in the PlugIns folder. + output_bundle = ctx.actions.declare_output(ctx.attrs.name + "." + ctx.attrs.extension) + bundle_parts = [ + _get_xctrunner_binary(ctx), + _get_uitest_bundle(ctx), + ] + _get_xctrunner_frameworks(ctx) + assemble_bundle( + ctx = ctx, + bundle = output_bundle, + info_plist_part = process_info_plist(ctx, override_input = None), + parts = bundle_parts, + swift_stdlib_args = None, + ) + + return [DefaultInfo(default_output = output_bundle)] + +def _get_uitest_bundle(ctx: AnalysisContext) -> AppleBundlePart: + return AppleBundlePart( + source = ctx.attrs.test_bundle[DefaultInfo].default_outputs[0], + destination = AppleBundleDestination("plugins"), + ) + +def _get_xctrunner_binary(ctx: AnalysisContext) -> AppleBundlePart: + platform_path = ctx.attrs._apple_toolchain[AppleToolchainInfo].platform_path + copied_binary = ctx.actions.declare_output(ctx.attrs.name) + xctrunner_path = cmd_args(platform_path, "Developer/Library/Xcode/Agents/XCTRunner.app/XCTRunner", delimiter = "/") + ctx.actions.run(["cp", "-PR", xctrunner_path, copied_binary.as_output()], category = "copy_xctrunner") + return AppleBundlePart( + source = copied_binary, + destination = AppleBundleDestination("executables"), + ) + +def _get_xctrunner_frameworks(ctx: AnalysisContext) -> list[AppleBundlePart]: + # We need to copy the framework as AppleBundlePart requires an artifact. + # It would be nicer to make this an arglike and avoid the copies. + # It would also be nicer to exclude the headers. + def copy_platform_framework(platform_relative_path: str) -> AppleBundlePart: + copied_framework = ctx.actions.declare_output(paths.basename(platform_relative_path)) + path = cmd_args(ctx.attrs._apple_toolchain[AppleToolchainInfo].platform_path, platform_relative_path, delimiter = "/") + ctx.actions.run(["cp", "-PR", path, copied_framework.as_output()], category = "copy_framework", identifier = platform_relative_path) + return AppleBundlePart( + source = copied_framework, + destination = AppleBundleDestination("frameworks"), + codesign_on_copy = True, + ) + + runner_frameworks = [ + "Developer/Library/Frameworks/XCTest.framework", + "Developer/Library/PrivateFrameworks/XCTAutomationSupport.framework", + "Developer/Library/PrivateFrameworks/XCTestCore.framework", + "Developer/Library/PrivateFrameworks/XCTestSupport.framework", + "Developer/Library/PrivateFrameworks/XCUIAutomation.framework", + "Developer/Library/PrivateFrameworks/XCUnit.framework", + ] + return [copy_platform_framework(p) for p in runner_frameworks] diff --git a/prelude/apple/swift/swift_compilation.bzl b/prelude/apple/swift/swift_compilation.bzl index e45f3d27fb934..f9570a5ce25c6 100644 --- a/prelude/apple/swift/swift_compilation.bzl +++ b/prelude/apple/swift/swift_compilation.bzl @@ -194,10 +194,7 @@ def compile_swift( exported_headers: list[CHeader], objc_modulemap_pp_info: [CPreprocessor, None], framework_search_paths_flags: cmd_args, - extra_search_paths_flags: list[ArgLike] = []) -> [SwiftCompilationOutput, None]: - if not srcs: - return None - + extra_search_paths_flags: list[ArgLike] = []) -> ([SwiftCompilationOutput, None], DefaultInfo): # If this target imports XCTest we need to pass the search path to its swiftmodule. framework_search_paths = cmd_args() framework_search_paths.add(_get_xctest_swiftmodule_search_path(ctx)) @@ -228,12 +225,7 @@ def compile_swift( else: compiled_underlying_pcm = None - toolchain = ctx.attrs._apple_toolchain[AppleToolchainInfo].swift_toolchain_info - module_name = get_module_name(ctx) - output_header = ctx.actions.declare_output(module_name + "-Swift.h") - - output_swiftmodule = ctx.actions.declare_output(module_name + SWIFTMODULE_EXTENSION) shared_flags = _get_shared_flags( ctx, @@ -246,6 +238,23 @@ def compile_swift( extra_search_paths_flags, ) shared_flags.add(framework_search_paths) + swift_interface_info = _create_swift_interface(ctx, shared_flags, module_name) + + if not srcs: + return (None, swift_interface_info) + + toolchain = ctx.attrs._apple_toolchain[AppleToolchainInfo].swift_toolchain_info + + if ctx.attrs.serialize_debugging_options: + if exported_headers: + # TODO(T99100029): We cannot use VFS overlays with Buck2, so we have to disable + # serializing debugging options for mixed libraries to debug successfully + warning("Mixed libraries cannot serialize debugging options, disabling for module `{}` in rule `{}`".format(module_name, ctx.label)) + elif not toolchain.prefix_serialized_debugging_options: + warning("The current toolchain does not support prefixing serialized debugging options, disabling for module `{}` in rule `{}`".format(module_name, ctx.label)) + + output_header = ctx.actions.declare_output(module_name + "-Swift.h") + output_swiftmodule = ctx.actions.declare_output(module_name + SWIFTMODULE_EXTENSION) if toolchain.can_toolchain_emit_obj_c_header_textually: _compile_swiftmodule(ctx, toolchain, shared_flags, srcs, output_swiftmodule, output_header) @@ -281,7 +290,7 @@ def compile_swift( pre = CPreprocessor(headers = [swift_header]) # Pass up the swiftmodule paths for this module and its exported_deps - return SwiftCompilationOutput( + return (SwiftCompilationOutput( output_map_artifact = object_output.output_map_artifact, object_files = object_output.object_files, object_format = toolchain.object_format, @@ -293,7 +302,7 @@ def compile_swift( swift_debug_info = extract_and_merge_swift_debug_infos(ctx, deps_providers, [output_swiftmodule]), clang_debug_info = extract_and_merge_clang_debug_infos(ctx, deps_providers), compilation_database = _create_compilation_database(ctx, srcs, object_output.argsfiles.absolute[SWIFT_EXTENSION]), - ) + ), swift_interface_info) # Swift headers are postprocessed to make them compatible with Objective-C # compilation that does not use -fmodules. This is a workaround for the bad @@ -505,19 +514,7 @@ def _get_shared_flags( else: cmd.add(["-enable-experimental-cxx-interop"]) - serialize_debugging_options = False - if ctx.attrs.serialize_debugging_options: - if objc_headers: - # TODO(T99100029): We cannot use VFS overlays with Buck2, so we have to disable - # serializing debugging options for mixed libraries to debug successfully - warning("Mixed libraries cannot serialize debugging options, disabling for module `{}` in rule `{}`".format(module_name, ctx.label)) - elif not toolchain.prefix_serialized_debugging_options: - warning("The current toolchain does not support prefixing serialized debugging options, disabling for module `{}` in rule `{}`".format(module_name, ctx.label)) - else: - # Apply the debug prefix map to Swift serialized debugging info. - # This will allow for debugging remotely built swiftmodule files. - serialize_debugging_options = True - + serialize_debugging_options = ctx.attrs.serialize_debugging_options and not objc_headers and toolchain.prefix_serialized_debugging_options if serialize_debugging_options: cmd.add([ "-Xfrontend", @@ -641,17 +638,18 @@ def _add_mixed_library_flags_to_cmd( if not objc_headers: return - # TODO(T99100029): We cannot use VFS overlays to mask this import from - # the debugger as they require absolute paths. Instead we will enforce - # that mixed libraries do not have serialized debugging info and rely on - # rdeps to serialize the correct paths. - for arg in objc_modulemap_pp_info.relative_args.args: - cmd.add("-Xcc") - cmd.add(arg) + if objc_modulemap_pp_info: + # TODO(T99100029): We cannot use VFS overlays to mask this import from + # the debugger as they require absolute paths. Instead we will enforce + # that mixed libraries do not have serialized debugging info and rely on + # rdeps to serialize the correct paths. + for arg in objc_modulemap_pp_info.relative_args.args: + cmd.add("-Xcc") + cmd.add(arg) - for arg in objc_modulemap_pp_info.modular_args: - cmd.add("-Xcc") - cmd.add(arg) + for arg in objc_modulemap_pp_info.modular_args: + cmd.add("-Xcc") + cmd.add(arg) cmd.add("-import-underlying-module") @@ -833,6 +831,47 @@ def _create_compilation_database( return SwiftCompilationDatabase(db = cdb_artifact, other_outputs = argfile.cmd_form) +def _create_swift_interface(ctx: AnalysisContext, shared_flags: cmd_args, module_name: str) -> DefaultInfo: + swift_toolchain = ctx.attrs._apple_toolchain[AppleToolchainInfo].swift_toolchain_info + swift_ide_test_tool = swift_toolchain.swift_ide_test_tool + if not swift_ide_test_tool: + return DefaultInfo() + mk_swift_interface = swift_toolchain.mk_swift_interface + + identifier = module_name + ".interface.swift" + + argsfile, _ = ctx.actions.write( + identifier + ".argsfile", + shared_flags, + allow_args = True, + ) + interface_artifact = ctx.actions.declare_output(identifier) + + mk_swift_args = cmd_args( + mk_swift_interface, + "--swift-ide-test-tool", + swift_ide_test_tool, + "--module", + module_name, + "--out", + interface_artifact.as_output(), + "--", + cmd_args(cmd_args(argsfile, format = "@{}", delimiter = "")).hidden([shared_flags]), + ) + + ctx.actions.run( + mk_swift_args, + category = "mk_swift_interface", + identifier = identifier, + ) + + return DefaultInfo( + default_output = interface_artifact, + other_outputs = [ + argsfile, + ], + ) + def _exported_deps(ctx) -> list[Dependency]: if ctx.attrs.reexport_all_header_dependencies: return ctx.attrs.exported_deps + ctx.attrs.deps diff --git a/prelude/apple/swift/swift_toolchain.bzl b/prelude/apple/swift/swift_toolchain.bzl index c3a7a0a128199..7e89e77436dc0 100644 --- a/prelude/apple/swift/swift_toolchain.bzl +++ b/prelude/apple/swift/swift_toolchain.bzl @@ -69,6 +69,8 @@ def swift_toolchain_impl(ctx): sdk_path = ctx.attrs._internal_sdk_path or ctx.attrs.sdk_path, swift_stdlib_tool = ctx.attrs.swift_stdlib_tool[RunInfo], swift_stdlib_tool_flags = ctx.attrs.swift_stdlib_tool_flags, + swift_ide_test_tool = ctx.attrs.swift_ide_test_tool[RunInfo] if ctx.attrs.swift_ide_test_tool else None, + mk_swift_interface = cmd_args(ctx.attrs._swiftc_wrapper[RunInfo]).add(ctx.attrs.make_swift_interface[RunInfo]), supports_relative_resource_dir = ctx.attrs.supports_relative_resource_dir, supports_swift_cxx_interoperability_mode = ctx.attrs.supports_swift_cxx_interoperability_mode, supports_swift_importing_objc_forward_declarations = ctx.attrs.supports_swift_importing_obj_c_forward_declarations, diff --git a/prelude/apple/swift/swift_toolchain_types.bzl b/prelude/apple/swift/swift_toolchain_types.bzl index e2f7c8241e7bc..10e6941c3d0c1 100644 --- a/prelude/apple/swift/swift_toolchain_types.bzl +++ b/prelude/apple/swift/swift_toolchain_types.bzl @@ -31,6 +31,8 @@ SwiftToolchainInfo = provider( "sdk_path": provider_field(typing.Any, default = None), "swift_stdlib_tool_flags": provider_field(typing.Any, default = None), "swift_stdlib_tool": provider_field(typing.Any, default = None), + "swift_ide_test_tool": provider_field(typing.Any, default = None), + "mk_swift_interface": provider_field(typing.Any, default = None), "runtime_run_paths": provider_field(typing.Any, default = None), # [str] "supports_relative_resource_dir": provider_field(typing.Any, default = None), # bool "supports_swift_cxx_interoperability_mode": provider_field(typing.Any, default = None), # bool diff --git a/prelude/apple/tools/BUCK.v2 b/prelude/apple/tools/BUCK.v2 index d86e174c4c00b..b31f120b5b324 100644 --- a/prelude/apple/tools/BUCK.v2 +++ b/prelude/apple/tools/BUCK.v2 @@ -41,6 +41,12 @@ python_bootstrap_binary( visibility = ["PUBLIC"], ) +python_bootstrap_binary( + name = "make_swift_interface", + main = "make_swift_interface.py", + visibility = ["PUBLIC"], +) + python_bootstrap_binary( name = "make_vfsoverlay", main = "make_vfsoverlay.py", diff --git a/prelude/apple/tools/bundling/action_metadata.py b/prelude/apple/tools/bundling/action_metadata.py index 8f73315f8f9cf..56569e8bf2a0e 100644 --- a/prelude/apple/tools/bundling/action_metadata.py +++ b/prelude/apple/tools/bundling/action_metadata.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from io import TextIOBase from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union _METADATA_VERSION = 1 @@ -27,7 +27,7 @@ class _Metadata: digests: List[_Item] -def _object_hook(dict: Dict[str, Any]) -> Any: +def _object_hook(dict: Dict[str, Any]) -> Union[_Item, _Metadata]: if "version" in dict: return _Metadata(**dict) else: diff --git a/prelude/apple/tools/bundling/assemble_bundle.py b/prelude/apple/tools/bundling/assemble_bundle.py index 17484001525b3..484d67045e097 100644 --- a/prelude/apple/tools/bundling/assemble_bundle.py +++ b/prelude/apple/tools/bundling/assemble_bundle.py @@ -9,7 +9,7 @@ import os import shutil from pathlib import Path -from typing import cast, Dict, List, Optional +from typing import Any, cast, Dict, List, Optional from .assemble_bundle_types import BundleSpecItem, IncrementalContext from .incremental_state import IncrementalState, IncrementalStateItem @@ -18,7 +18,7 @@ should_assemble_incrementally, ) -_LOGGER = logging.getLogger(__name__) +_LOGGER: logging.Logger = logging.getLogger(__name__) def assemble_bundle( @@ -27,36 +27,23 @@ def assemble_bundle( incremental_context: Optional[IncrementalContext], check_conflicts: bool, ) -> Optional[List[IncrementalStateItem]]: - # It's possible to have the same spec multiple times as different - # apple_resource() targets can refer to the _same_ resource file. - # - # On RE, we're not allowed to overwrite files, so prevent doing - # identical file copies. - # - # Do not reorder spec items to achieve determinism. - # Rely on the fact that `dict` preserves key order. - deduplicated_spec = list(dict.fromkeys(spec)) - # Force same sorting as in Buck1 for `SourcePathWithAppleBundleDestination` - # WARNING: This logic is tightly coupled with how spec filtering is done in `_filter_conflicting_paths` method during incremental bundling. Don't change unless you fully understand what is going on here. - deduplicated_spec.sort() - incremental_result = None if incremental_context: - if should_assemble_incrementally(deduplicated_spec, incremental_context): + if should_assemble_incrementally(spec, incremental_context): incremental_result = _assemble_incrementally( bundle_path, - deduplicated_spec, + spec, incremental_context.metadata, cast(IncrementalState, incremental_context.state), check_conflicts, ) else: - _assemble_non_incrementally(bundle_path, deduplicated_spec, check_conflicts) + _assemble_non_incrementally(bundle_path, spec, check_conflicts) incremental_result = calculate_incremental_state( - deduplicated_spec, incremental_context.metadata + spec, incremental_context.metadata ) else: - _assemble_non_incrementally(bundle_path, deduplicated_spec, check_conflicts) + _assemble_non_incrementally(bundle_path, spec, check_conflicts) # External tooling (e.g., Xcode) might depend on the timestamp of the bundle bundle_path.touch() @@ -76,9 +63,9 @@ def _assemble_non_incrementally( logging.getLogger(__name__).info("Assembling bundle non-incrementally.") _cleanup_output(incremental=False, path=bundle_path) - copied_contents = {} + copied_contents: Dict[Path, str] = {} - def _copy(src, dst, **kwargs) -> None: + def _copy(src: str, dst: Path, **kwargs: Any) -> None: if check_conflicts: if dst in copied_contents: raise RuntimeError( diff --git a/prelude/apple/tools/bundling/assemble_bundle_types.py b/prelude/apple/tools/bundling/assemble_bundle_types.py index 2f0ea75970482..c304c4f09bca2 100644 --- a/prelude/apple/tools/bundling/assemble_bundle_types.py +++ b/prelude/apple/tools/bundling/assemble_bundle_types.py @@ -5,6 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +from __future__ import annotations + import functools from dataclasses import dataclass from pathlib import Path @@ -23,21 +25,21 @@ class BundleSpecItem: dst: str codesign_on_copy: bool = False - def __eq__(self, other) -> bool: + def __eq__(self: BundleSpecItem, other: Optional[BundleSpecItem]) -> bool: return ( - other + other is not None and self.src == other.src and self.dst == other.dst and self.codesign_on_copy == other.codesign_on_copy ) - def __ne__(self, other) -> bool: + def __ne__(self: BundleSpecItem, other: BundleSpecItem) -> bool: return not self.__eq__(other) - def __hash__(self) -> int: + def __hash__(self: BundleSpecItem) -> int: return hash((self.src, self.dst, self.codesign_on_copy)) - def __lt__(self, other) -> bool: + def __lt__(self: BundleSpecItem, other: BundleSpecItem) -> bool: return ( self.src < other.src or self.dst < other.dst diff --git a/prelude/apple/tools/bundling/incremental_state.py b/prelude/apple/tools/bundling/incremental_state.py index d35daecf58a28..e2bd67fbbe4bf 100644 --- a/prelude/apple/tools/bundling/incremental_state.py +++ b/prelude/apple/tools/bundling/incremental_state.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from io import TextIOBase from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from apple.tools.code_signing.codesign_bundle import CodesignConfiguration @@ -49,7 +49,7 @@ class IncrementalState: class IncrementalStateJSONEncoder(json.JSONEncoder): - def default(self, o: Any) -> Any: + def default(self, o: object) -> object: if isinstance(o, IncrementalState): return { "items": [self.default(i) for i in o.items], @@ -76,7 +76,7 @@ def default(self, o: Any) -> Any: return super().default(o) -def _object_hook(dict: Dict[str, Any]) -> Any: +def _object_hook(dict: Dict[str, Any]) -> Union[IncrementalState, IncrementalStateItem]: if "version" in dict: dict["codesign_on_copy_paths"] = [ Path(p) for p in dict.pop("codesign_on_copy_paths") diff --git a/prelude/apple/tools/bundling/incremental_utils.py b/prelude/apple/tools/bundling/incremental_utils.py index a92b6463f0876..df5af7584e281 100644 --- a/prelude/apple/tools/bundling/incremental_utils.py +++ b/prelude/apple/tools/bundling/incremental_utils.py @@ -80,7 +80,7 @@ def should_assemble_incrementally( def _codesigned_on_copy_paths_from_previous_build_which_are_present_in_current_build( previously_codesigned_on_copy_paths: Set[Path], all_input_files: Set[Path], -): +) -> Set[Path]: all_input_files_and_directories = all_input_files | { i for file in all_input_files for i in file.parents } diff --git a/prelude/apple/tools/bundling/main.py b/prelude/apple/tools/bundling/main.py index 0bf0cb1d6591a..f9d54ad9824a4 100644 --- a/prelude/apple/tools/bundling/main.py +++ b/prelude/apple/tools/bundling/main.py @@ -13,18 +13,16 @@ import shlex import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional from apple.tools.code_signing.apple_platform import ApplePlatform from apple.tools.code_signing.codesign_bundle import ( AdhocSigningContext, codesign_bundle, CodesignConfiguration, - non_adhoc_signing_context, -) -from apple.tools.code_signing.list_codesign_identities_command_factory import ( - ListCodesignIdentitiesCommandFactory, + signing_context_with_profile_selection, ) +from apple.tools.code_signing.list_codesign_identities import ListCodesignIdentities from apple.tools.re_compatibility_utils.writable import make_dir_recursively_writable @@ -257,27 +255,39 @@ def _main() -> None: pr.enable() if args.codesign: - assert args.info_plist_source and args.info_plist_destination and args.platform + if not args.info_plist_source: + raise RuntimeError( + "Paths to Info.plist source file should be set when code signing is required." + ) + if not args.info_plist_destination: + raise RuntimeError( + "Info.plist destination path should be set when code signing is required." + ) + if not args.platform: + raise RuntimeError( + "Apple platform should be set when code signing is required." + ) if args.ad_hoc: signing_context = AdhocSigningContext( codesign_identity=args.ad_hoc_codesign_identity ) selected_identity_argument = args.ad_hoc_codesign_identity else: - assert ( - args.profiles_dir - ), "Path to directory with provisioning profile files should be set when signing is not ad-hoc." - signing_context = non_adhoc_signing_context( + if not args.profiles_dir: + raise RuntimeError( + "Path to directory with provisioning profile files should be set when signing is not ad-hoc." + ) + signing_context = signing_context_with_profile_selection( info_plist_source=args.info_plist_source, info_plist_destination=args.info_plist_destination, provisioning_profiles_dir=args.profiles_dir, entitlements_path=args.entitlements, platform=args.platform, - list_codesign_identities_command_factory=ListCodesignIdentitiesCommandFactory.override( + list_codesign_identities=ListCodesignIdentities.override( shlex.split(args.codesign_identities_command) ) if args.codesign_identities_command - else None, + else ListCodesignIdentities.default(), log_file_path=args.log_file, ) selected_identity_argument = ( @@ -289,6 +299,7 @@ def _main() -> None: with args.spec.open(mode="rb") as spec_file: spec = json.load(spec_file, object_hook=lambda d: BundleSpecItem(**d)) + spec = _deduplicate_spec(spec) incremental_context = _incremental_context( incremenatal_state_path=args.incremental_state, @@ -452,7 +463,7 @@ def _write_incremental_state( codesign_configuration: CodesignConfiguration, selected_codesign_identity: Optional[str], swift_stdlib_paths: List[Path], -): +) -> None: state = IncrementalState( items, codesigned=codesigned, @@ -470,6 +481,22 @@ def _write_incremental_state( raise +def _deduplicate_spec(spec: List[BundleSpecItem]) -> List[BundleSpecItem]: + # It's possible to have the same spec multiple times as different + # apple_resource() targets can refer to the _same_ resource file. + # + # On RE, we're not allowed to overwrite files, so prevent doing + # identical file copies. + # + # Do not reorder spec items to achieve determinism. + # Rely on the fact that `dict` preserves key order. + deduplicated_spec = list(dict.fromkeys(spec)) + # Force same sorting as in Buck1 for `SourcePathWithAppleBundleDestination` + # WARNING: This logic is tightly coupled with how spec filtering is done in `_filter_conflicting_paths` method during incremental bundling. Don't change unless you fully understand what is going on here. + deduplicated_spec.sort() + return deduplicated_spec + + def _setup_logging( stderr_level: int, file_level: int, log_path: Optional[Path] ) -> None: @@ -497,7 +524,7 @@ def _setup_logging( class ColoredLogFormatter(logging.Formatter): - _colors = { + _colors: Dict[int, str] = { logging.DEBUG: "\x1b[m", logging.INFO: "\x1b[37m", logging.WARNING: "\x1b[33m", @@ -506,10 +533,10 @@ class ColoredLogFormatter(logging.Formatter): } _reset_color = "\x1b[0m" - def __init__(self, text_format: str): + def __init__(self, text_format: str) -> None: self.text_format = text_format - def format(self, record: logging.LogRecord): + def format(self, record: logging.LogRecord) -> str: colored_format = ( self._colors[record.levelno] + self.text_format + self._reset_color ) diff --git a/prelude/apple/tools/code_signing/app_id.py b/prelude/apple/tools/code_signing/app_id.py index fbd70e5171495..2e0e0b3b3ab00 100644 --- a/prelude/apple/tools/code_signing/app_id.py +++ b/prelude/apple/tools/code_signing/app_id.py @@ -22,11 +22,11 @@ class _ReGroupName(str, Enum): team_id = "team_id" bundle_id = "bundle_id" - _re_string = "^(?P<{team_id}>[A-Z0-9]{{10}})\\.(?P<{bundle_id}>.+)$".format( + _re_string: str = "^(?P<{team_id}>[A-Z0-9]{{10}})\\.(?P<{bundle_id}>.+)$".format( team_id=_ReGroupName.team_id, bundle_id=_ReGroupName.bundle_id, ) - _re_pattern = re.compile(_re_string) + _re_pattern: re.Pattern[str] = re.compile(_re_string) # Takes a application identifier and splits it into Team ID and bundle ID. # Prefix is always a ten-character alphanumeric sequence. Bundle ID may be a fully-qualified name or a wildcard ending in *. diff --git a/prelude/apple/tools/code_signing/codesign_bundle.py b/prelude/apple/tools/code_signing/codesign_bundle.py index 9303d6a7444ad..174440ee13912 100644 --- a/prelude/apple/tools/code_signing/codesign_bundle.py +++ b/prelude/apple/tools/code_signing/codesign_bundle.py @@ -26,12 +26,8 @@ ICodesignCommandFactory, ) from .fast_adhoc import is_fast_adhoc_codesign_allowed, should_skip_adhoc_signing_path -from .identity import CodeSigningIdentity from .info_plist_metadata import InfoPlistMetadata -from .list_codesign_identities_command_factory import ( - IListCodesignIdentitiesCommandFactory, - ListCodesignIdentitiesCommandFactory, -) +from .list_codesign_identities import IListCodesignIdentities from .prepare_code_signing_entitlements import prepare_code_signing_entitlements from .prepare_info_plist import prepare_info_plist from .provisioning_profile_diagnostics import ( @@ -54,7 +50,7 @@ DefaultReadProvisioningProfileCommandFactory() ) -_LOGGER = logging.getLogger(__name__) +_LOGGER: logging.Logger = logging.getLogger(__name__) def _select_provisioning_profile( @@ -62,11 +58,11 @@ def _select_provisioning_profile( provisioning_profiles_dir: Path, entitlements_path: Optional[Path], platform: ApplePlatform, - list_codesign_identities_command_factory: IListCodesignIdentitiesCommandFactory, + list_codesign_identities: IListCodesignIdentities, read_provisioning_profile_command_factory: IReadProvisioningProfileCommandFactory = _default_read_provisioning_profile_command_factory, log_file_path: Optional[Path] = None, ) -> SelectedProvisioningProfileInfo: - identities = _list_identities(list_codesign_identities_command_factory) + identities = list_codesign_identities.list_codesign_identities() provisioning_profiles = _read_provisioning_profiles( provisioning_profiles_dir, read_provisioning_profile_command_factory ) @@ -92,6 +88,7 @@ def _select_provisioning_profile( diagnostics=mismatches, bundle_id=info_plist_metadata.bundle_id, provisioning_profiles_dir=provisioning_profiles_dir, + identities=identities, log_file_path=log_file_path, ) ) @@ -102,29 +99,27 @@ def _select_provisioning_profile( class AdhocSigningContext: codesign_identity: str - def __init__(self, codesign_identity: Optional[str] = None): + def __init__(self, codesign_identity: Optional[str] = None) -> None: self.codesign_identity = codesign_identity or "-" @dataclass -class NonAdhocSigningContext: +class SigningContextWithProfileSelection: info_plist_source: Path info_plist_destination: Path info_plist_metadata: InfoPlistMetadata selected_profile_info: SelectedProvisioningProfileInfo -def non_adhoc_signing_context( +def signing_context_with_profile_selection( info_plist_source: Path, info_plist_destination: Path, provisioning_profiles_dir: Path, entitlements_path: Optional[Path], platform: ApplePlatform, - list_codesign_identities_command_factory: Optional[ - IListCodesignIdentitiesCommandFactory - ] = None, + list_codesign_identities: IListCodesignIdentities, log_file_path: Optional[Path] = None, -) -> NonAdhocSigningContext: +) -> SigningContextWithProfileSelection: with open(info_plist_source, mode="rb") as info_plist_file: info_plist_metadata = InfoPlistMetadata.from_file(info_plist_file) selected_profile_info = _select_provisioning_profile( @@ -132,12 +127,11 @@ def non_adhoc_signing_context( provisioning_profiles_dir=provisioning_profiles_dir, entitlements_path=entitlements_path, platform=platform, - list_codesign_identities_command_factory=list_codesign_identities_command_factory - or ListCodesignIdentitiesCommandFactory.default(), + list_codesign_identities=list_codesign_identities, log_file_path=log_file_path, ) - return NonAdhocSigningContext( + return SigningContextWithProfileSelection( info_plist_source, info_plist_destination, info_plist_metadata, @@ -153,16 +147,16 @@ class CodesignConfiguration(str, Enum): def codesign_bundle( bundle_path: Path, - signing_context: Union[AdhocSigningContext, NonAdhocSigningContext], + signing_context: Union[AdhocSigningContext, SigningContextWithProfileSelection], entitlements_path: Optional[Path], platform: ApplePlatform, - codesign_on_copy_paths: List[Path], + codesign_on_copy_paths: List[str], codesign_args: List[str], codesign_tool: Optional[Path] = None, codesign_configuration: Optional[CodesignConfiguration] = None, ) -> None: with tempfile.TemporaryDirectory() as tmp_dir: - if isinstance(signing_context, NonAdhocSigningContext): + if isinstance(signing_context, SigningContextWithProfileSelection): info_plist_metadata = signing_context.info_plist_metadata selected_profile_info = signing_context.selected_profile_info prepared_entitlements_path = prepare_code_signing_entitlements( @@ -224,28 +218,50 @@ def codesign_bundle( ) -def _list_identities( - list_codesign_identities_command_factory: IListCodesignIdentitiesCommandFactory, -) -> List[CodeSigningIdentity]: - output = subprocess.check_output( - list_codesign_identities_command_factory.list_codesign_identities_command(), - encoding="utf-8", - ) - return CodeSigningIdentity.parse_security_stdout(output) - - def _read_provisioning_profiles( dirpath: Path, read_provisioning_profile_command_factory: IReadProvisioningProfileCommandFactory, ) -> List[ProvisioningProfileMetadata]: - return [ - _provisioning_profile_from_file_path( - dirpath / f, - read_provisioning_profile_command_factory, - ) + paths = [ + dirpath / f for f in os.listdir(dirpath) if (f.endswith(".mobileprovision") or f.endswith(".provisionprofile")) ] + with tempfile.TemporaryDirectory() as tmp_dir: + path_to_data = _decode_provisioning_profiles( + paths, tmp_dir, read_provisioning_profile_command_factory + ) + return [ + ProvisioningProfileMetadata.from_provisioning_profile_file_content(path, data) + for path, data in path_to_data.items() + ] + + +def _decode_provisioning_profiles( + paths: List[Path], + tmp_dir: str, + read_provisioning_profile_command_factory: IReadProvisioningProfileCommandFactory, +) -> Dict[Path, bytes]: + """Reads multiple provisioning profiles in parallel.""" + processes: Dict[Path, ParallelProcess] = {} + result = {} + with ExitStack() as stack: + for path in paths: + command = read_provisioning_profile_command_factory.read_provisioning_profile_command( + path + ) + process = _spawn_process( + command=command, + tmp_dir=tmp_dir, + stack=stack, + pipe_stdout=True, + ) + processes[path] = process + for path, process in processes.items(): + data, _ = process.process.communicate() + process.check_result() + result[path] = data + return result def _provisioning_profile_from_file_path( @@ -272,7 +288,7 @@ def _read_entitlements_file(path: Optional[Path]) -> Optional[Dict[str, Any]]: def _dry_codesign_everything( bundle_path: Path, - codesign_on_copy_paths: List[Path], + codesign_on_copy_paths: List[str], identity_fingerprint: str, tmp_dir: str, codesign_tool: Path, @@ -321,7 +337,7 @@ def _dry_codesign_everything( def _codesign_everything( bundle_path: Path, - codesign_on_copy_paths: List[Path], + codesign_on_copy_paths: List[str], identity_fingerprint: str, tmp_dir: str, codesign_command_factory: ICodesignCommandFactory, @@ -367,20 +383,47 @@ def _codesign_everything( @dataclass -class CodesignProcess: - process: subprocess.Popen - stdout_path: str +class ParallelProcess: + process: subprocess.Popen[bytes] + stdout_path: Optional[str] stderr_path: str def check_result(self) -> None: if self.process.returncode == 0: return - with open(self.stdout_path, encoding="utf8") as stdout, open( - self.stderr_path, encoding="utf8" - ) as stderr: - raise RuntimeError( - "\nstdout:\n{}\n\nstderr:\n{}\n".format(stdout.read(), stderr.read()) + with ExitStack() as stack: + stderr = stack.enter_context(open(self.stderr_path, encoding="utf8")) + stderr_string = f"\nstderr:\n{stderr.read()}\n" + stdout = ( + stack.enter_context(open(self.stdout_path, encoding="utf8")) + if self.stdout_path + else None ) + stdout_string = f"\nstdout:\n{stdout.read()}\n" if stdout else "" + raise RuntimeError(f"{stdout_string}{stderr_string}") + + +def _spawn_process( + command: List[Union[str, Path]], + tmp_dir: str, + stack: ExitStack, + pipe_stdout: bool = False, +) -> ParallelProcess: + if pipe_stdout: + stdout_path = None + stdout = subprocess.PIPE + else: + stdout_path = os.path.join(tmp_dir, uuid.uuid4().hex) + stdout = stack.enter_context(open(stdout_path, "w")) + stderr_path = os.path.join(tmp_dir, uuid.uuid4().hex) + stderr = stack.enter_context(open(stderr_path, "w")) + _LOGGER.info(f"Executing command: {command}") + process = subprocess.Popen(command, stdout=stdout, stderr=stderr) + return ParallelProcess( + process, + stdout_path, + stderr_path, + ) def _spawn_codesign_process( @@ -391,21 +434,11 @@ def _spawn_codesign_process( entitlements: Optional[Path], stack: ExitStack, codesign_args: List[str], -) -> CodesignProcess: - stdout_path = os.path.join(tmp_dir, uuid.uuid4().hex) - stdout = stack.enter_context(open(stdout_path, "w")) - stderr_path = os.path.join(tmp_dir, uuid.uuid4().hex) - stderr = stack.enter_context(open(stderr_path, "w")) +) -> ParallelProcess: command = codesign_command_factory.codesign_command( path, identity_fingerprint, entitlements, codesign_args ) - _LOGGER.info(f"Executing codesign command: {command}") - process = subprocess.Popen(command, stdout=stdout, stderr=stderr) - return CodesignProcess( - process, - stdout_path, - stderr_path, - ) + return _spawn_process(command=command, tmp_dir=tmp_dir, stack=stack) def _codesign_paths( @@ -418,7 +451,7 @@ def _codesign_paths( codesign_args: List[str], ) -> None: """Codesigns several paths in parallel.""" - processes: List[CodesignProcess] = [] + processes: List[ParallelProcess] = [] with ExitStack() as stack: for path in paths: process = _spawn_codesign_process( diff --git a/prelude/apple/tools/code_signing/codesign_command_factory.py b/prelude/apple/tools/code_signing/codesign_command_factory.py index 894b07a14ec31..bf77cbfb17c49 100644 --- a/prelude/apple/tools/code_signing/codesign_command_factory.py +++ b/prelude/apple/tools/code_signing/codesign_command_factory.py @@ -23,9 +23,10 @@ def codesign_command( class DefaultCodesignCommandFactory(ICodesignCommandFactory): - _command_args = ["--force", "--sign"] + codesign_tool: Path + _command_args: List[str] = ["--force", "--sign"] - def __init__(self, codesign_tool: Optional[Path]): + def __init__(self, codesign_tool: Optional[Path]) -> None: self.codesign_tool = codesign_tool or Path("codesign") def codesign_command( @@ -47,7 +48,10 @@ def codesign_command( class DryRunCodesignCommandFactory(ICodesignCommandFactory): - def __init__(self, codesign_tool: Path): + codesign_tool: Path + codesign_on_copy_file_paths: Optional[List[Path]] + + def __init__(self, codesign_tool: Path) -> None: self.codesign_tool = codesign_tool self.codesign_on_copy_file_paths = None @@ -64,7 +68,8 @@ def codesign_command( args = [path, "--identity", identity_fingerprint] if entitlements: args += ["--entitlements", entitlements] if entitlements else [] - if self.codesign_on_copy_file_paths: + codesign_on_copy_file_paths = self.codesign_on_copy_file_paths + if codesign_on_copy_file_paths: args += ["--extra-paths-to-sign"] - args += self.codesign_on_copy_file_paths + args += codesign_on_copy_file_paths return [self.codesign_tool] + args diff --git a/prelude/apple/tools/code_signing/fast_adhoc.py b/prelude/apple/tools/code_signing/fast_adhoc.py index 8d3fb16c6b027..e752232da98cf 100644 --- a/prelude/apple/tools/code_signing/fast_adhoc.py +++ b/prelude/apple/tools/code_signing/fast_adhoc.py @@ -11,11 +11,11 @@ import sys from pathlib import Path -from typing import Optional +from typing import List, Optional, Union from .apple_platform import ApplePlatform -_LOGGER = logging.getLogger(__name__) +_LOGGER: logging.Logger = logging.getLogger(__name__) def _find_executable_for_signed_path(path: Path, platform: ApplePlatform) -> Path: @@ -29,7 +29,9 @@ def _find_executable_for_signed_path(path: Path, platform: ApplePlatform) -> Pat return contents_dir / path.stem -def _logged_subprocess_run(name, why, args): +def _logged_subprocess_run( + name: str, why: str, args: List[Union[str, Path]] +) -> subprocess.CompletedProcess[str]: _LOGGER.info(f" Calling {name} to {why}: `{args}`") result = subprocess.run( args, @@ -74,7 +76,7 @@ def should_skip_adhoc_signing_path( identity_fingerprint: str, entitlements_path: Optional[Path], platform: ApplePlatform, -): +) -> bool: logging.getLogger(__name__).info( f"Checking if should skip adhoc signing path `{path}` with identity `{identity_fingerprint}` and entitlements `{entitlements_path}` for platform `{platform}`" ) diff --git a/prelude/apple/tools/code_signing/identity.py b/prelude/apple/tools/code_signing/identity.py index 7893a6ff9af83..ed6ba58278b86 100644 --- a/prelude/apple/tools/code_signing/identity.py +++ b/prelude/apple/tools/code_signing/identity.py @@ -22,12 +22,12 @@ class _ReGroupName(str, Enum): fingerprint = "fingerprint" subject_common_name = "subject_common_name" - _re_string = '(?P<{fingerprint}>[A-F0-9]{{40}}) "(?P<{subject_common_name}>.+)"(?!.*CSSMERR_.+)'.format( + _re_string: str = '(?P<{fingerprint}>[A-F0-9]{{40}}) "(?P<{subject_common_name}>.+)"(?!.*CSSMERR_.+)'.format( fingerprint=_ReGroupName.fingerprint, subject_common_name=_ReGroupName.subject_common_name, ) - _pattern = re.compile(_re_string) + _pattern: re.Pattern[str] = re.compile(_re_string) @classmethod def parse_security_stdout(cls, text: str) -> List[CodeSigningIdentity]: diff --git a/prelude/apple/tools/code_signing/info_plist_metadata.py b/prelude/apple/tools/code_signing/info_plist_metadata.py index 21942eecbb5df..75f666fba0b85 100644 --- a/prelude/apple/tools/code_signing/info_plist_metadata.py +++ b/prelude/apple/tools/code_signing/info_plist_metadata.py @@ -20,7 +20,7 @@ class InfoPlistMetadata: is_watchos_app: bool @staticmethod - def from_file(info_plist_file: IO) -> InfoPlistMetadata: + def from_file(info_plist_file: IO[bytes]) -> InfoPlistMetadata: root = detect_format_and_load(info_plist_file) return InfoPlistMetadata( root["CFBundleIdentifier"], diff --git a/prelude/apple/tools/code_signing/list_codesign_identities.py b/prelude/apple/tools/code_signing/list_codesign_identities.py new file mode 100644 index 0000000000000..b072ebb478d8c --- /dev/null +++ b/prelude/apple/tools/code_signing/list_codesign_identities.py @@ -0,0 +1,49 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +from __future__ import annotations + +import subprocess + +from abc import ABCMeta, abstractmethod +from typing import List + +from .identity import CodeSigningIdentity + + +class IListCodesignIdentities(metaclass=ABCMeta): + @abstractmethod + def list_codesign_identities(self) -> List[CodeSigningIdentity]: + raise NotImplementedError + + +class ListCodesignIdentities(IListCodesignIdentities): + _default_command = ["security", "find-identity", "-v", "-p", "codesigning"] + + def __init__(self, command: List[str]) -> None: + self.command = command + + @classmethod + def default(cls) -> IListCodesignIdentities: + return cls(cls._default_command) + + @classmethod + def override(cls, command: List[str]) -> IListCodesignIdentities: + return cls(command) + + def list_codesign_identities(self) -> List[CodeSigningIdentity]: + return _list_identities(self.command) + + +def _list_identities( + command: List[str], +) -> List[CodeSigningIdentity]: + output = subprocess.check_output( + command, + encoding="utf-8", + ) + return CodeSigningIdentity.parse_security_stdout(output) diff --git a/prelude/apple/tools/code_signing/list_codesign_identities_command_factory.py b/prelude/apple/tools/code_signing/list_codesign_identities_command_factory.py deleted file mode 100644 index ad92e239b99d1..0000000000000 --- a/prelude/apple/tools/code_signing/list_codesign_identities_command_factory.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under both the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree and the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. - -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from typing import List - - -class IListCodesignIdentitiesCommandFactory(metaclass=ABCMeta): - @abstractmethod - def list_codesign_identities_command(self) -> List[str]: - raise NotImplementedError - - -class ListCodesignIdentitiesCommandFactory(IListCodesignIdentitiesCommandFactory): - _default_command = ["security", "find-identity", "-v", "-p", "codesigning"] - - def __init__(self, command: List[str]): - self.command = command - - @classmethod - def default(cls) -> ListCodesignIdentitiesCommandFactory: - return cls(cls._default_command) - - @classmethod - def override(cls, command: List[str]) -> ListCodesignIdentitiesCommandFactory: - return cls(command) - - def list_codesign_identities_command(self) -> List[str]: - return self.command diff --git a/prelude/apple/tools/code_signing/main.py b/prelude/apple/tools/code_signing/main.py index 549e324990b79..c0faf2d74b610 100644 --- a/prelude/apple/tools/code_signing/main.py +++ b/prelude/apple/tools/code_signing/main.py @@ -13,8 +13,9 @@ from .codesign_bundle import ( AdhocSigningContext, codesign_bundle, - non_adhoc_signing_context, + signing_context_with_profile_selection, ) +from .list_codesign_identities import ListCodesignIdentities from .provisioning_profile_selection import CodeSignProvisioningError @@ -86,7 +87,7 @@ def decorate_error_message(message: str) -> str: return " ".join(["❗️", message]) -def _main(): +def _main() -> None: args = _args_parser().parse_args() try: if args.ad_hoc: @@ -97,11 +98,12 @@ def _main(): assert ( args.profiles_dir ), "Path to directory with provisioning profile files should be set when signing is not ad-hoc." - signing_context = non_adhoc_signing_context( + signing_context = signing_context_with_profile_selection( info_plist_source=args.bundle_path / args.info_plist, info_plist_destination=args.info_plist, provisioning_profiles_dir=args.profiles_dir, entitlements_path=args.entitlements, + list_codesign_identities=ListCodesignIdentities.default(), platform=args.platform, ) codesign_bundle( diff --git a/prelude/apple/tools/code_signing/provisioning_profile_diagnostics.py b/prelude/apple/tools/code_signing/provisioning_profile_diagnostics.py index a94b1da8d6a41..625a850a71c8d 100644 --- a/prelude/apple/tools/code_signing/provisioning_profile_diagnostics.py +++ b/prelude/apple/tools/code_signing/provisioning_profile_diagnostics.py @@ -11,6 +11,8 @@ from .apple_platform import ApplePlatform +from .identity import CodeSigningIdentity + from .provisioning_profile_metadata import ProvisioningProfileMetadata META_IOS_DEVELOPER_CERTIFICATE_LINK: str = "https://www.internalfb.com/intern/qa/5198/how-do-i-get-the-fb-ios-developer-certificate" @@ -23,7 +25,7 @@ class IProvisioningProfileDiagnostics(metaclass=ABCMeta): profile: ProvisioningProfileMetadata - def __init__(self, profile: ProvisioningProfileMetadata): + def __init__(self, profile: ProvisioningProfileMetadata) -> None: self.profile = profile @abstractmethod @@ -40,7 +42,7 @@ def __init__( profile: ProvisioningProfileMetadata, team_id: str, team_id_constraint: str, - ): + ) -> None: super().__init__(profile) self.team_id = team_id self.team_id_constraint = team_id_constraint @@ -58,7 +60,7 @@ def __init__( profile: ProvisioningProfileMetadata, bundle_id: str, bundle_id_constraint: str, - ): + ) -> None: super().__init__(profile) self.bundle_id = bundle_id self.bundle_id_constraint = bundle_id_constraint @@ -74,7 +76,7 @@ def __init__( self, profile: ProvisioningProfileMetadata, bundle_id_match_length: int, - ): + ) -> None: super().__init__(profile) self.bundle_id_match_length = bundle_id_match_length @@ -91,7 +93,7 @@ def __init__( profile: ProvisioningProfileMetadata, bundle_id_match_length: int, platform_constraint: ApplePlatform, - ): + ) -> None: super().__init__(profile) self.bundle_id_match_length = bundle_id_match_length self.platform_constraint = platform_constraint @@ -112,7 +114,7 @@ def __init__( bundle_id_match_length: int, mismatched_key: str, mismatched_value: str, - ): + ) -> None: super().__init__(profile) self.bundle_id_match_length = bundle_id_match_length self.mismatched_key = mismatched_key @@ -129,7 +131,7 @@ def __init__( self, profile: ProvisioningProfileMetadata, bundle_id_match_length: int, - ): + ) -> None: super().__init__(profile) self.bundle_id_match_length = bundle_id_match_length @@ -147,6 +149,7 @@ def interpret_provisioning_profile_diagnostics( diagnostics: List[IProvisioningProfileDiagnostics], bundle_id: str, provisioning_profiles_dir: Path, + identities: List[CodeSigningIdentity], log_file_path: Optional[Path] = None, ) -> str: if not diagnostics: @@ -182,10 +185,16 @@ def find_mismatch(class_type: Type[_T]) -> Optional[_T]: ) if mismatch := find_mismatch(DeveloperCertificateMismatch): + identities_description = ( + "WARNING: NO SIGNING IDENTITIES FOUND!" + if len(identities) == 0 + else f"List of signing identities: `{identities}`." + ) return "".join( [ header, f"The provisioning profile `{mismatch.profile.file_path.name}` satisfies all constraints, but no matching certificates were found in your keychain. ", + identities_description, f"Please download and install the latest certificate from {META_IOS_DEVELOPER_CERTIFICATE_LINK}.", footer, ] diff --git a/prelude/apple/tools/code_signing/provisioning_profile_metadata.py b/prelude/apple/tools/code_signing/provisioning_profile_metadata.py index 331ded4e7e2e3..d8b05ad731e8d 100644 --- a/prelude/apple/tools/code_signing/provisioning_profile_metadata.py +++ b/prelude/apple/tools/code_signing/provisioning_profile_metadata.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Dict, Set +from typing import Any, Dict, FrozenSet, Set from apple.tools.plistlib_utils import detect_format_and_loads @@ -30,13 +30,15 @@ class ProvisioningProfileMetadata: developer_certificate_fingerprints: Set[str] entitlements: Dict[str, Any] - _mergeable_entitlements_keys = { - "application-identifier", - "beta-reports-active", - "get-task-allow", - "com.apple.developer.aps-environment", - "com.apple.developer.team-identifier", - } + _mergeable_entitlements_keys: FrozenSet[str] = frozenset( + [ + "application-identifier", + "beta-reports-active", + "get-task-allow", + "com.apple.developer.aps-environment", + "com.apple.developer.team-identifier", + ] + ) # See `ProvisioningProfileMetadataFactory::getAppIDFromEntitlements` from `ProvisioningProfileMetadataFactory.java` in Buck v1 def get_app_id(self) -> AppId: diff --git a/prelude/apple/tools/code_signing/provisioning_profile_selection.py b/prelude/apple/tools/code_signing/provisioning_profile_selection.py index 0858efb873477..7358b2d53fddd 100644 --- a/prelude/apple/tools/code_signing/provisioning_profile_selection.py +++ b/prelude/apple/tools/code_signing/provisioning_profile_selection.py @@ -25,7 +25,7 @@ ) from .provisioning_profile_metadata import ProvisioningProfileMetadata -_LOGGER = logging.getLogger(__name__) +_LOGGER: logging.Logger = logging.getLogger(__name__) class CodeSignProvisioningError(Exception): @@ -45,8 +45,8 @@ def _parse_team_id_from_entitlements( def _matches_or_array_is_subset_of( entitlement_name: str, - expected_value: Any, - actual_value: Any, + expected_value: object, + actual_value: object, platform: ApplePlatform, ) -> bool: if expected_value is None: @@ -170,7 +170,7 @@ def select_best_provisioning_profile( result = None # Used for error messages - diagnostics = [] + diagnostics: List[IProvisioningProfileDiagnostics] = [] def log_mismatched_profile(mismatch: IProvisioningProfileDiagnostics) -> None: diagnostics.append(mismatch) diff --git a/prelude/apple/tools/code_signing/read_provisioning_profile_command_factory.py b/prelude/apple/tools/code_signing/read_provisioning_profile_command_factory.py index 0d4d7536231c5..ed5b01a4def92 100644 --- a/prelude/apple/tools/code_signing/read_provisioning_profile_command_factory.py +++ b/prelude/apple/tools/code_signing/read_provisioning_profile_command_factory.py @@ -27,6 +27,7 @@ class DefaultReadProvisioningProfileCommandFactory( "der", "-verify", "-noverify", + "-nosigs", "-in", ] diff --git a/prelude/apple/tools/dry_codesign_tool.py b/prelude/apple/tools/dry_codesign_tool.py index 5e445b3acbfc7..71364ddb76ed0 100644 --- a/prelude/apple/tools/dry_codesign_tool.py +++ b/prelude/apple/tools/dry_codesign_tool.py @@ -48,7 +48,7 @@ def _args_parser() -> argparse.ArgumentParser: return parser -def _main(): +def _main() -> None: args = _args_parser().parse_args() content = { # This is always empty string if you check `DryCodeSignStep` class usages in buck1 diff --git a/prelude/apple/tools/info_plist_processor/main.py b/prelude/apple/tools/info_plist_processor/main.py index b1d3e6b67008b..157652f60faff 100644 --- a/prelude/apple/tools/info_plist_processor/main.py +++ b/prelude/apple/tools/info_plist_processor/main.py @@ -19,7 +19,9 @@ class _SubcommandName(str, Enum): process = "process" -def _create_preprocess_subparser(subparsers): +def _create_preprocess_subparser( + subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]", +) -> None: parser = subparsers.add_parser( _SubcommandName.preprocess.value, description="Sub-command to expand macro variables in parametrized Info.plist files. It's the Buck v2 equivalent of what `FindAndReplaceStep` and `InfoPlistSubstitution` do.", @@ -53,7 +55,9 @@ def _create_preprocess_subparser(subparsers): ) -def _create_process_subparser(subparsers): +def _create_process_subparser( + subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]", +) -> None: parser = subparsers.add_parser( _SubcommandName.process.value, description="Sub-command to do the final processing of the Info.plist before it's copied to the application bundle. It's the Buck v2 equivalent of what `PlistProcessStep` does in v1.", @@ -92,7 +96,7 @@ def _create_process_subparser(subparsers): ) -def _parse_args(): +def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Tool to process Info.plist file before it is placed into the bundle. It's the Buck v2 equivalent of what `AppleInfoPlist` build rule from v1 does." ) @@ -102,7 +106,7 @@ def _parse_args(): return parser.parse_args() -def main(): +def main() -> None: args = _parse_args() if args.subcommand_name == _SubcommandName.preprocess: with ExitStack() as stack: diff --git a/prelude/apple/tools/info_plist_processor/preprocess.py b/prelude/apple/tools/info_plist_processor/preprocess.py index 18b8e71a9c1f4..36cf9b2317585 100644 --- a/prelude/apple/tools/info_plist_processor/preprocess.py +++ b/prelude/apple/tools/info_plist_processor/preprocess.py @@ -8,6 +8,7 @@ import json import re from enum import Enum +from typing import Dict, TextIO class _ReGroupName(str, Enum): @@ -17,7 +18,7 @@ class _ReGroupName(str, Enum): closeparen = "closeparen" -_re_string = "\\$(?P<{openparen}>[\\{{\\(])(?P<{variable}>[^\\}}\\):]+)(?::(?P<{modifier}>[^\\}}\\)]+))?(?P<{closeparen}>[\\}}\\)])".format( +_re_string: str = "\\$(?P<{openparen}>[\\{{\\(])(?P<{variable}>[^\\}}\\):]+)(?::(?P<{modifier}>[^\\}}\\)]+))?(?P<{closeparen}>[\\}}\\)])".format( openparen=_ReGroupName.openparen, variable=_ReGroupName.variable, modifier=_ReGroupName.modifier, @@ -25,7 +26,9 @@ class _ReGroupName(str, Enum): ) -def _make_substitution_dict(substitutions_json_file, product_name): +def _make_substitution_dict( + substitutions_json_file: TextIO, product_name: str +) -> Dict[str, str]: result = { "EXECUTABLE_NAME": product_name, "PRODUCT_NAME": product_name, @@ -36,7 +39,9 @@ def _make_substitution_dict(substitutions_json_file, product_name): return result -def _process_line(line, pattern, substitutions): +def _process_line( + line: str, pattern: re.Pattern[str], substitutions: Dict[str, str] +) -> str: result = line pos = 0 substituted_keys = set() @@ -62,7 +67,12 @@ def _process_line(line, pattern, substitutions): return result -def preprocess(input_file, output_file, substitutions_file, product_name): +def preprocess( + input_file: TextIO, + output_file: TextIO, + substitutions_file: TextIO, + product_name: str, +) -> None: pattern = re.compile(_re_string) substitutions = _make_substitution_dict(substitutions_file, product_name) for line in input_file: diff --git a/prelude/apple/tools/info_plist_processor/process.py b/prelude/apple/tools/info_plist_processor/process.py index 91f1d5d38653b..bca05d93a75b1 100644 --- a/prelude/apple/tools/info_plist_processor/process.py +++ b/prelude/apple/tools/info_plist_processor/process.py @@ -7,7 +7,7 @@ import json import plistlib -from typing import Any, Dict, IO, Optional +from typing import Any, Dict, IO, Optional, TextIO from apple.tools.plistlib_utils import detect_format_and_load @@ -26,12 +26,12 @@ def _merge_plist_dicts( def process( - input_file: IO, - output_file: IO, - override_input_file: Optional[IO] = None, + input_file: IO[bytes], + output_file: IO[bytes], + override_input_file: Optional[IO[bytes]] = None, additional_keys: Optional[Dict[str, Any]] = None, - additional_keys_file: Optional[IO] = None, - override_keys_file: Optional[IO] = None, + additional_keys_file: Optional[TextIO] = None, + override_keys_file: Optional[TextIO] = None, output_format: plistlib.PlistFormat = plistlib.FMT_BINARY, ) -> None: root = detect_format_and_load(input_file) diff --git a/prelude/apple/tools/ipa_package_maker.py b/prelude/apple/tools/ipa_package_maker.py index 710872b94fa4a..f4848dfa4f867 100644 --- a/prelude/apple/tools/ipa_package_maker.py +++ b/prelude/apple/tools/ipa_package_maker.py @@ -19,13 +19,13 @@ from apple.tools.re_compatibility_utils.writable import make_dir_recursively_writable -def _copy_ipa_contents(ipa_contents_dir: Path, output_dir: Path): +def _copy_ipa_contents(ipa_contents_dir: Path, output_dir: Path) -> None: if os.path.exists(output_dir): shutil.rmtree(output_dir, ignore_errors=False) shutil.copytree(ipa_contents_dir, output_dir, symlinks=True, dirs_exist_ok=False) -def _delete_empty_SwiftSupport_dir(output_dir: Path): +def _delete_empty_SwiftSupport_dir(output_dir: Path) -> None: swiftSupportDir = output_dir / "SwiftSupport" if not swiftSupportDir.exists(): return @@ -46,7 +46,7 @@ def _package_ipa_contents( compression_level: int, validator: Optional[Path], validator_args: List[str], -): +) -> None: with tempfile.TemporaryDirectory() as processed_package_dir: processed_package_dir_path = Path(processed_package_dir) _copy_ipa_contents(ipa_contents_dir, processed_package_dir_path) @@ -86,7 +86,7 @@ def _package_ipa_contents( ) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description="Tool to make an .ipa package file.") parser.add_argument( "--ipa-contents-dir", diff --git a/prelude/apple/tools/make_swift_interface.py b/prelude/apple/tools/make_swift_interface.py new file mode 100755 index 0000000000000..13c91db7e4024 --- /dev/null +++ b/prelude/apple/tools/make_swift_interface.py @@ -0,0 +1,282 @@ +#!/usr/bin/env fbpython +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +""" +Remaps swiftc arguments to be usable by swift-ide-test, and invokes +swift-ide-test with said arguments. +""" + +import argparse +import dataclasses +import optparse +import pathlib +import subprocess as proc +import sys + +from contextlib import contextmanager +from typing import Iterable, List, Optional + + +@dataclasses.dataclass +class SwiftIDETestArguments(object): + sdk: Optional[str] + target: Optional[str] + xcc: Iterable[str] + defines: Iterable[str] + frameworks: Iterable[str] + includes: Iterable[str] + resource_dir: str + enable_cxx_interop: bool + cxx_interoperability_mode: Optional[str] + upcoming_features: Iterable[str] + explicit_swift_module_map: Optional[str] + swift_version: Optional[str] + + def to_args(self) -> List[str]: + args = [] + if self.target: + args.append("--target") + args.append(self.target) + + if self.sdk: + args.append("--sdk") + args.append(self.sdk) + + for define in self.defines: + args.append("-D") + args.append(define) + + for include in self.includes: + args.append("-I") + args.append(include) + + for framework in self.frameworks: + args.append("-F") + args.append(framework) + + for xcc in self.xcc: + args.append("--Xcc") + args.append(xcc) + + args.append("--resource-dir") + args.append(self.resource_dir) + + if self.enable_cxx_interop: + args.append("-enable-experimental-cxx-interop") + + if self.cxx_interoperability_mode: + # swift-ide-test only understands -enable-experimental-cxx-interop, + # not the versioned -cxx-interoperability-mode=. + args.append("-enable-experimental-cxx-interop") + + if self.upcoming_features: + for feature in self.upcoming_features: + args.append("-enable-upcoming-feature") + args.append(feature) + + if self.explicit_swift_module_map: + args.append("--explicit-swift-module-map-file") + args.append(self.explicit_swift_module_map) + + if self.swift_version: + args.append("-swift-version") + args.append(self.swift_version) + return args + + +class LongSingleDashOpt(optparse.Option): + """ + This Option subclass allows for long arguments specified with single dashes, + e.g. -sdk (the default implementation only allows long options with two + dashes) + """ + + def _set_opt_strings(self, opts): + for opt in opts: + if len(opt) < 2: + raise optparse.OptionError( + "invalid option string %r: " + "must be at least two characters long" % opt, + self, + ) + elif len(opt) == 2: + self._short_opts.append(opt) + else: + self._long_opts.append(opt) + + +class IgnoreUnknownLongSingleDashOptParser(optparse.OptionParser): + """ + This OptionParser subclass allows for + (a) long arguments specified with single dashes (e.g. -sdk) + (b) ignoring unknown arguments + The default OptionParser doesn't have either of these behaviors. + """ + + def __init__(self, *args, **kwargs): + kwargs["option_class"] = LongSingleDashOpt + super().__init__(*args, **kwargs) + + def _process_args(self, largs, rargs, values): + while rargs: + try: + arg = rargs[0] + if arg == "--": + del rargs[0] + return + elif arg[0:2] == "--": + self._process_long_opt(rargs, values) + elif arg[:1] == "-" and len(arg) > 1: + if len(arg) > 2: + self._process_long_opt(rargs, values) + else: + self._process_short_opts(rargs, values) + elif self.allow_interspersed_args: + largs.append(arg) + del rargs[0] + else: + return + except optparse.BadOptionError: + continue + + +def parse_swiftc_args(arguments: List[str]) -> SwiftIDETestArguments: # noqa: C901 + """ + We can't use argparse to do our parsing because arguments like -Xcc + need to accept arguments that are prefixed with `-`. + + optparse can handle this, and it's only soft deprecated (i.e. it should + stay around, just not actively developed), so we should be safe to use it. + + Additionally, our subclasses above are safe, since optparse is no longer + actively developed. + """ + parser = IgnoreUnknownLongSingleDashOptParser() + + parser.add_option("-sdk", dest="sdk") + parser.add_option("-target", dest="target") + parser.add_option("-Xcc", action="append", default=[], dest="xcc") + parser.add_option("-D", dest="defines", action="append", default=[]) + parser.add_option("-F", dest="frameworks", action="append", default=[]) + parser.add_option("-I", dest="includes", action="append", default=[]) + parser.add_option("-resource-dir", dest="resource_dir") + parser.add_option( + "-enable-experimental-cxx-interop", + action="store_true", + default=False, + dest="enable_experimental_cxx_interop", + ) + parser.add_option("-Xfrontend", action="append", default=[], dest="xfrontend") + parser.add_option("-swift-version", dest="swift_version") + parser.add_option("-cxx-interoperability-mode", dest="cxx_interoperability_mode") + + options, leftovers = parser.parse_args(arguments) + + frontend_parser = IgnoreUnknownLongSingleDashOptParser() + frontend_parser.add_option( + "-enable-upcoming-feature", + dest="enable_upcoming_feature", + action="append", + default=[], + ) + frontend_parser.add_option( + "-explicit-swift-module-map-file", dest="explicit_swift_module_map" + ) + frontend_options = frontend_parser.parse_args(options.xfrontend)[0] + + resource_dir = options.resource_dir + if not resource_dir: + # If an explicit resource dir was not provided, we need to figure out + # which resource id would have been used, which, in the case of Xcode, + # is relative to the swiftc used. + assert len(leftovers) >= 1 + compiler_path = pathlib.Path(leftovers[0]) + assert compiler_path.name == "swiftc" + resource_dir_path = compiler_path.parents[1] / "lib" / "swift" + assert resource_dir_path.exists() + resource_dir = str(resource_dir_path) + + return SwiftIDETestArguments( + options.sdk, + options.target, + options.xcc, + options.defines, + options.frameworks, + options.includes, + resource_dir, + options.enable_experimental_cxx_interop, + options.cxx_interoperability_mode, + frontend_options.enable_upcoming_feature, + frontend_options.explicit_swift_module_map, + options.swift_version, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Uses swift-ide-test to generate a swift interface", + fromfile_prefix_chars="@", + ) + parser.add_argument( + "--swift-ide-test-tool", + required=True, + help="Path to swift-ide-test binary.", + ) + parser.add_argument( + "--module", + required=True, + help="Name of the module to generate the interface for.", + ) + parser.add_argument( + "--out", + help="Path to output file.", + default="-", + ) + parser.add_argument( + "arguments", + nargs="*", + default=[], + help="File containing compiler arguments to use to invoke" + + " swift-ide-test. Note these arguments should be in the format CC" + + " expects, not swift-ide-test, as this tool converts the arguments" + + " as needed", + ) + return parser.parse_args() + + +@contextmanager +def open_or_stdout(out): + if out == "-": + yield sys.stdout + else: + with open(out, "w") as f: + yield f + + +def main() -> None: + args = parse_args() + + parsed = parse_swiftc_args(args.arguments) + with open_or_stdout(args.out) as out: + proc.run( + [ + args.swift_ide_test_tool, + "--source-filename=x", + "--print-module", + "--module-to-print", + args.module, + "--module-print-submodules", + ] + + parsed.to_args(), + stdout=out, + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/prelude/apple/tools/plistlib_utils.py b/prelude/apple/tools/plistlib_utils.py index 2f927c38ce510..63ea7a3566393 100644 --- a/prelude/apple/tools/plistlib_utils.py +++ b/prelude/apple/tools/plistlib_utils.py @@ -7,13 +7,14 @@ import plistlib from io import BytesIO +from typing import Any, Dict, IO -def _is_fmt_binary(header): +def _is_fmt_binary(header: bytes) -> bool: return header[:8] == b"bplist00" -def detect_format_and_load(fp): +def detect_format_and_load(fp: IO[bytes]) -> Dict[str, Any]: header = fp.read(32) fp.seek(0) if _is_fmt_binary(header): @@ -23,6 +24,6 @@ def detect_format_and_load(fp): return plistlib.load(fp, fmt=fmt) -def detect_format_and_loads(value): +def detect_format_and_loads(value: bytes) -> Dict[str, Any]: fp = BytesIO(value) return detect_format_and_load(fp) diff --git a/prelude/apple/tools/re_compatibility_utils/writable.py b/prelude/apple/tools/re_compatibility_utils/writable.py index 668f96ebc4062..7fc1aec5a6bdf 100644 --- a/prelude/apple/tools/re_compatibility_utils/writable.py +++ b/prelude/apple/tools/re_compatibility_utils/writable.py @@ -10,7 +10,7 @@ import stat -def make_path_user_writable(path: str): +def make_path_user_writable(path: str) -> None: # On Linux, `os.chmod()` does not support setting the permissions on a symlink. # `chmod` manpage says: # > AT_SYMLINK_NOFOLLOW If pathname is a symbolic link, do not @@ -26,7 +26,7 @@ def make_path_user_writable(path: str): os.chmod(path, st.st_mode | stat.S_IWUSR, follow_symlinks=follow_symlinks) -def make_dir_recursively_writable(dir: str): +def make_dir_recursively_writable(dir: str) -> None: for dirpath, _, filenames in os.walk(dir): make_path_user_writable(dirpath) for filename in filenames: diff --git a/prelude/apple/tools/resource_broker/BUCK.v2 b/prelude/apple/tools/resource_broker/BUCK.v2 new file mode 100644 index 0000000000000..5aef86d064fb3 --- /dev/null +++ b/prelude/apple/tools/resource_broker/BUCK.v2 @@ -0,0 +1,32 @@ +python_binary( + name = "resource_broker", + main = "main.py", + visibility = ["PUBLIC"], + deps = [ + ":main", + ], +) + +python_library( + name = "main", + srcs = ["main.py"], + deps = [ + ":lib", + ], +) + +python_library( + name = "lib", + srcs = glob( + [ + "*.py", + ], + exclude = [ + "main.py", + ], + ), + deps = [ + "fbsource//third-party/pypi/dataclasses-json:dataclasses-json", + "fbsource//third-party/pypi/packaging:packaging", + ], +) diff --git a/prelude/apple/tools/resource_broker/idb_companion.py b/prelude/apple/tools/resource_broker/idb_companion.py new file mode 100644 index 0000000000000..f831e32cc9070 --- /dev/null +++ b/prelude/apple/tools/resource_broker/idb_companion.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import os +import signal +from dataclasses import dataclass +from io import TextIOWrapper + + +@dataclass +class IdbCompanion: + socket_address: str + pid: int + stderr: TextIOWrapper + + def cleanup(self) -> None: + os.kill(self.pid, signal.SIGTERM) + self.stderr.close() diff --git a/prelude/apple/tools/resource_broker/idb_target.py b/prelude/apple/tools/resource_broker/idb_target.py new file mode 100644 index 0000000000000..37de481dc1f08 --- /dev/null +++ b/prelude/apple/tools/resource_broker/idb_target.py @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import json +from dataclasses import dataclass +from enum import Enum +from typing import List, Optional + +from dataclasses_json import dataclass_json + + +class SimState(str, Enum): + booted = "Booted" + shutdown = "Shutdown" + + +@dataclass_json +@dataclass +class IdbTarget: + name: str + os_version: str + udid: str + state: SimState + host: str = "" + port: int = 0 + + +def managed_simulators_from_stdout(stdout: Optional[str]) -> List[IdbTarget]: + if not stdout: + return [] + targets = map( + # pyre-ignore[16]: `from_dict` is dynamically provided by `dataclass_json` + IdbTarget.from_dict, + json.loads(stdout), + ) + return list(targets) diff --git a/prelude/apple/tools/resource_broker/ios.py b/prelude/apple/tools/resource_broker/ios.py new file mode 100644 index 0000000000000..379367c08e364 --- /dev/null +++ b/prelude/apple/tools/resource_broker/ios.py @@ -0,0 +1,204 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import os +from typing import List, Optional + +from packaging.version import Version + +from .idb_companion import IdbCompanion + +from .idb_target import IdbTarget, managed_simulators_from_stdout, SimState + +from .simctl_runtime import list_ios_runtimes, XCSimRuntime + +from .timeouts import SIMULATOR_BOOT_TIMEOUT + +from .utils import ( + execute_generic_text_producing_command, + spawn_companion, + wait_for_idb_companions, +) + + +def _device_set_path() -> str: + return os.path.expanduser("~/Library/Developer/Buck2IdbDeviceSet") + + +def _list_managed_simulators_command(simulator_manager: str) -> List[str]: + return [ + simulator_manager, + "list", + "--device-set-path", + _device_set_path(), + "--only", + "simulator", + ] + + +def _create_simulator_command(simulator_manager: str, sim_spec: str) -> List[str]: + return [ + simulator_manager, + "create", + "--device-set-path", + _device_set_path(), + "--configuration", + sim_spec, + ] + + +def _boot_simulator_command(simulator_manager: str, udid: str) -> List[str]: + return [ + simulator_manager, + "boot", + "--device-set-path", + _device_set_path(), + udid, + ] + + +def _compatible_device_type_from_runtime(runtime: XCSimRuntime) -> Optional[str]: + iphones = filter( + lambda t: t.product_family == "iPhone", runtime.supported_device_types + ) + if not iphones: + return None + default = next(iphones) + return next( + (device_type.name for device_type in iphones if device_type.name == "iPhone 8"), + default.name, + ) + + +def _select_latest_simulator_spec(runtimes: List[XCSimRuntime]) -> str: + runtimes.sort(key=lambda x: Version(x.version), reverse=True) + for runtime in runtimes: + device_type = _compatible_device_type_from_runtime(runtime) + if device_type: + return f"{device_type},{runtime.name}" + raise RuntimeError( + "No XCode simctl compatible iOS runtime and device available. Try to `sudo xcode-select -s ` and *open Xcode to install all required components*." + ) + + +def _spawn_companion_for_simulator_command( + udid: str, grpc_domain_sock: str +) -> List[str]: + return [ + "idb_companion", + "--device-set-path", + _device_set_path(), + "--udid", + udid, + "--only", + "simulator", + "--grpc-domain-sock", + grpc_domain_sock, + ] + + +async def _generic_managed_simulators_command( + name: str, cmd: List[str] +) -> List[IdbTarget]: + stdout = await execute_generic_text_producing_command(name=name, cmd=cmd) + return managed_simulators_from_stdout(stdout) + + +async def _list_managed_simulators(simulator_manager: str) -> List[IdbTarget]: + list_cmd = _list_managed_simulators_command(simulator_manager=simulator_manager) + return await _generic_managed_simulators_command( + name="list managed simulators", cmd=list_cmd + ) + + +async def _create_simulator(simulator_manager: str) -> List[IdbTarget]: + runtimes = await list_ios_runtimes() + spec = _select_latest_simulator_spec(runtimes) + create_cmd = _create_simulator_command( + simulator_manager=simulator_manager, sim_spec=spec + ) + return await _generic_managed_simulators_command( + name="create simulators", cmd=create_cmd + ) + + +async def _get_managed_simulators_create_if_needed( + simulator_manager: str, +) -> List[IdbTarget]: + managed_simulators = await _list_managed_simulators( + simulator_manager=simulator_manager + ) + if managed_simulators: + return managed_simulators + + managed_simulators = await _create_simulator(simulator_manager=simulator_manager) + if managed_simulators: + return managed_simulators + + raise RuntimeError( + "Failed to create an iOS simulator. Try to `sudo xcode-select -s ` and *open Xcode to install all required components*." + ) + + +def _select_simulator( + only_booted: bool, all_simulators: List[IdbTarget] +) -> Optional[IdbTarget]: + return next( + filter( + lambda s: s.state == SimState.booted if only_booted else True, + iter(all_simulators), + ), + None, + ) + + +def _select_simulator_with_preference( + prefer_booted: bool, all_simulators: List[IdbTarget] +) -> IdbTarget: + simulator = _select_simulator( + only_booted=prefer_booted, all_simulators=all_simulators + ) + if not simulator and prefer_booted: + simulator = _select_simulator(only_booted=False, all_simulators=all_simulators) + if not simulator: + raise RuntimeError("Expected at least unbooted simulator entity to be selected") + return simulator + + +async def _ios_simulator(simulator_manager: str, booted: bool) -> List[IdbCompanion]: + managed_simulators = await _get_managed_simulators_create_if_needed( + simulator_manager=simulator_manager + ) + simulator = _select_simulator_with_preference( + prefer_booted=booted, all_simulators=managed_simulators + ) + if simulator.state != SimState.booted and booted: + boot_cmd = _boot_simulator_command( + simulator_manager=simulator_manager, udid=simulator.udid + ) + await execute_generic_text_producing_command( + name="boot simulator", + cmd=boot_cmd, + timeout=SIMULATOR_BOOT_TIMEOUT, + ) + + grpc_domain_sock = f"/tmp/buck2_idb_companion_{simulator.udid}" + process = await spawn_companion( + command=_spawn_companion_for_simulator_command( + simulator.udid, grpc_domain_sock + ), + log_file_suffix=f"companion_launch_logs_for_{simulator.udid}.log", + ) + return await wait_for_idb_companions([process]) + + +async def ios_unbooted_simulator(simulator_manager: str) -> List[IdbCompanion]: + return await _ios_simulator(simulator_manager=simulator_manager, booted=False) + + +async def ios_booted_simulator(simulator_manager: str) -> List[IdbCompanion]: + return await _ios_simulator(simulator_manager=simulator_manager, booted=True) diff --git a/prelude/apple/tools/resource_broker/macos.py b/prelude/apple/tools/resource_broker/macos.py new file mode 100644 index 0000000000000..d3aeaa4f92035 --- /dev/null +++ b/prelude/apple/tools/resource_broker/macos.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import asyncio +from typing import cast, List + +from .idb_companion import IdbCompanion + +from .utils import IdbCompanionProcess, spawn_companion, wait_for_idb_companions + + +def _boot_macos_companion_command(grpc_domain_sock: str) -> List[str]: + return [ + "idb_companion", + "--udid", + "mac", + "--grpc-domain-sock", + grpc_domain_sock, + ] + + +async def macos_idb_companions() -> List[IdbCompanion]: + addresses = [(i, f"/tmp/buck2_idb_companion_mac_{i}") for i in range(10)] + awaitables = [ + spawn_companion( + command=_boot_macos_companion_command(addr), + log_file_suffix=f"macos_companion_{i}.log", + ) + for i, addr in addresses + ] + results = await asyncio.gather(*awaitables, return_exceptions=True) + + if exception := next(filter(lambda r: isinstance(r, BaseException), results), None): + [r.cleanup() for r in results if isinstance(r, IdbCompanionProcess)] + raise cast(BaseException, exception) + + return await wait_for_idb_companions(cast(List[IdbCompanionProcess], results)) diff --git a/prelude/apple/tools/resource_broker/main.py b/prelude/apple/tools/resource_broker/main.py new file mode 100644 index 0000000000000..0600a351efb59 --- /dev/null +++ b/prelude/apple/tools/resource_broker/main.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import argparse +import asyncio +import json +import os +import signal +import sys +from enum import Enum +from time import sleep +from typing import List, Optional + +from .idb_companion import IdbCompanion + +from .ios import ios_booted_simulator, ios_unbooted_simulator + +from .macos import macos_idb_companions + +idb_companions: List[IdbCompanion] = [] + + +def _args_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Utility which helps to set up IDB companions which are used later by buck2 it runs tests locally." + ) + parser.add_argument( + "--simulator-manager", + required=False, + type=str, + help="Tool to manage simulators and their lifecycle. Required for iOS testing", + ) + parser.add_argument( + "--type", + metavar="", + type=_ResourceType, + choices=[e.value for e in _ResourceType], + required=True, + help=f""" + Type of required resources. + Pass `{_ResourceType.iosUnbootedSimulator}` to get a companion for iOS unbooted simulator. + Pass `{_ResourceType.iosBootedSimulator}` to get a companion for iOS booted simulator. + Pass `{_ResourceType.macosIdbCompanion}` to get MacOS companions. + """, + ) + return parser + + +class _ResourceType(str, Enum): + iosUnbootedSimulator = "ios_unbooted_simulator" + iosBootedSimulator = "ios_booted_simulator" + macosIdbCompanion = "macos_idb_companion" + + +def _exit_gracefully(*args: List[object]) -> None: + for idb_companion in idb_companions: + idb_companion.cleanup() + exit(0) + + +def _check_simulator_manager_exists(simulator_manager: Optional[str]) -> None: + if not simulator_manager: + raise Exception("Simulator manager is not specified") + + +def main() -> None: + args = _args_parser().parse_args() + if args.type == _ResourceType.iosBootedSimulator: + _check_simulator_manager_exists(args.simulator_manager) + idb_companions.extend(asyncio.run(ios_booted_simulator(args.simulator_manager))) + elif args.type == _ResourceType.iosUnbootedSimulator: + _check_simulator_manager_exists(args.simulator_manager) + idb_companions.extend( + asyncio.run(ios_unbooted_simulator(args.simulator_manager)) + ) + elif args.type == _ResourceType.macosIdbCompanion: + idb_companions.extend(asyncio.run(macos_idb_companions())) + pid = os.fork() + if pid == 0: + # child + signal.signal(signal.SIGINT, _exit_gracefully) + signal.signal(signal.SIGTERM, _exit_gracefully) + while True: + sleep(0.1) + else: + # Do not leak open FDs in parent + for c in idb_companions: + c.stderr.close() + result = { + "pid": pid, + "resources": [{"socket_address": c.socket_address} for c in idb_companions], + } + json.dump(result, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/prelude/apple/tools/resource_broker/simctl_runtime.py b/prelude/apple/tools/resource_broker/simctl_runtime.py new file mode 100644 index 0000000000000..55a5740d2778d --- /dev/null +++ b/prelude/apple/tools/resource_broker/simctl_runtime.py @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import json +from dataclasses import dataclass, field +from typing import List, Optional + +from dataclasses_json import config, dataclass_json + +from .utils import execute_generic_text_producing_command + + +@dataclass_json +@dataclass +class XCSimDevice: + name: str + product_family: str = field(metadata=config(field_name="productFamily")) + + +@dataclass_json +@dataclass +class XCSimRuntime: + name: str + version: str + supported_device_types: List[XCSimDevice] = field( + metadata=config(field_name="supportedDeviceTypes") + ) + + +@dataclass_json +@dataclass +class _XCSimRuntimes: + runtimes: List[XCSimRuntime] + + +def _list_ios_runtimes_command() -> List[str]: + return [ + "xcrun", + "simctl", + "list", + "runtimes", + "iOS", + "available", + "--json", + ] + + +def _simctl_runtimes_from_stdout(stdout: Optional[str]) -> List[XCSimRuntime]: + if not stdout: + return [] + data = json.loads(stdout) + # pyre-ignore[16]: `from_dict` is dynamically provided by `dataclass_json` + return _XCSimRuntimes.from_dict(data).runtimes + + +async def list_ios_runtimes() -> List[XCSimRuntime]: + stdout = await execute_generic_text_producing_command( + name="list iOS runtimes", cmd=_list_ios_runtimes_command() + ) + return _simctl_runtimes_from_stdout(stdout) diff --git a/prelude/genrule_types.bzl b/prelude/apple/tools/resource_broker/timeouts.py similarity index 60% rename from prelude/genrule_types.bzl rename to prelude/apple/tools/resource_broker/timeouts.py index 0793c705d4800..01804468760f6 100644 --- a/prelude/genrule_types.bzl +++ b/prelude/apple/tools/resource_broker/timeouts.py @@ -5,8 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -# A provider that's used as a marker for `genrule()`, allows dependents -# to distinguish such outputs -GenruleMarkerInfo = provider(fields = {}) +DEFAULT_OPERATION_TIMEOUT = 10 -GENRULE_MARKER_SUBTARGET_NAME = "genrule_marker" +# Simulator boot is an expensive command and can take a long time to complete +# depending on machine configuration and current machine load. +SIMULATOR_BOOT_TIMEOUT = 90 diff --git a/prelude/apple/tools/resource_broker/utils.py b/prelude/apple/tools/resource_broker/utils.py new file mode 100644 index 0000000000000..5128fd19bcad9 --- /dev/null +++ b/prelude/apple/tools/resource_broker/utils.py @@ -0,0 +1,140 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import asyncio +import json +import shlex +from dataclasses import dataclass +from io import TextIOWrapper +from pathlib import Path +from typing import Any, List, Tuple + +from dataclasses_json import dataclass_json + +from .idb_companion import IdbCompanion +from .timeouts import DEFAULT_OPERATION_TIMEOUT + + +@dataclass_json +@dataclass +class _IdbStdout: + grpc_path: str + + +@dataclass +class IdbCompanionProcess: + process: asyncio.subprocess.Process + stderr: TextIOWrapper + stderr_path: Path + + def cleanup(self) -> None: + self.process.terminate() + self.stderr.close() + + +async def _read_until_valid_json(stream: asyncio.StreamReader) -> object: + buffer = b"" + while True: + data = await stream.readuntil(b"}") + buffer += data + try: + return json.loads(buffer.decode()) + except json.JSONDecodeError: + pass + raise RuntimeError( + "Should not be reachable since either the valid JSON is there or `asyncio.IncompleteReadError` is raised." + ) + + +async def _read_stdout(p: IdbCompanionProcess) -> Tuple[int, TextIOWrapper, object]: + if not p.process.stdout: + raise ValueError("Expected stdout to be set for idb companion launch process.") + try: + json = await _read_until_valid_json(p.process.stdout) + except asyncio.IncompleteReadError as e: + if not e.partial: + with open(p.stderr_path) as f: + lines = f.readlines() + raise RuntimeError( + f"idb companion terminated unexpectedly with the following stderr:\n{lines}" + ) from e + else: + raise + return p.process.pid, p.stderr, json + + +async def wait_for_idb_companions( + processes: List[IdbCompanionProcess], + timeout: float = DEFAULT_OPERATION_TIMEOUT, +) -> List[IdbCompanion]: + reads = [asyncio.Task(_read_stdout(p)) for p in processes] + done, pending = await asyncio.wait( + reads, + timeout=timeout, + ) + if not pending: + results = [task.result() for task in done] + return [ + IdbCompanion( + # pyre-ignore[16]: `from_dict` is dynamically provided by `dataclass_json` + socket_address=_IdbStdout.from_dict(json_dict).grpc_path, + pid=pid, + stderr=stderr, + ) + for pid, stderr, json_dict in results + ] + + process_index = {reads[i]: processes[i] for i in range(len(processes))} + + stderr_paths = [] + + for task in pending: + task.cancel() + process_info = process_index[task] + stderr_paths.append(str(process_info.stderr_path)) + process_info.process.terminate() + + raise RuntimeError( + f"Timeout when trying to launch idb companions. List of files with stderr for pending companions: {stderr_paths}" + ) + + +async def execute_generic_text_producing_command( + name: str, cmd: List[str], timeout: float = DEFAULT_OPERATION_TIMEOUT +) -> str: + process = await asyncio.create_subprocess_exec( + *cmd, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) + if process.returncode != 0: + raise RuntimeError( + f"Failed to {name} with command:\n```\n{shlex.join(cmd)}\n```\nstdout:\n```\n{stdout.decode(errors='ignore')}\n```\nstdout:\n```\n{stderr.decode(errors='ignore')}\n```\n" + ) + return stdout.decode() + + +async def spawn_companion( + command: List[str], + log_file_suffix: str, +) -> IdbCompanionProcess: + stderr_path = Path("/tmp/buck2_idb_companion_logs") / f"stderr-{log_file_suffix}" + stderr_path.parent.mkdir(parents=True, exist_ok=True) + stderr = stderr_path.open(mode="w") + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=stderr, + ) + return IdbCompanionProcess( + process=process, + stderr=stderr, + stderr_path=stderr_path, + ) diff --git a/prelude/apple/tools/selective_debugging/macho.py b/prelude/apple/tools/selective_debugging/macho.py index 966bee5671083..9b4964891584c 100644 --- a/prelude/apple/tools/selective_debugging/macho.py +++ b/prelude/apple/tools/selective_debugging/macho.py @@ -20,7 +20,7 @@ class MachO: - def __str__(self): + def __str__(self) -> str: props = {} for k, v in self.__dict__.items(): props[k] = hex(v) @@ -39,7 +39,7 @@ class MachOHeader(MachO): reserved: int @property - def is_valid(self): + def is_valid(self) -> bool: return self.magic in (MH_CIGAM_64, MH_MAGIC_64) diff --git a/prelude/apple/tools/selective_debugging/main.py b/prelude/apple/tools/selective_debugging/main.py index 20920c02a0a9c..aa5e2b4afbe4f 100644 --- a/prelude/apple/tools/selective_debugging/main.py +++ b/prelude/apple/tools/selective_debugging/main.py @@ -12,7 +12,7 @@ from .scrubber import scrub -def _parse_args(): +def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Tool to postprocess executables/dylibs." ) @@ -38,7 +38,7 @@ def _parse_args(): return parser.parse_args() -def main(): +def main() -> None: args = _parse_args() try: scrub( diff --git a/prelude/apple/tools/selective_debugging/scrubber.py b/prelude/apple/tools/selective_debugging/scrubber.py index f0d4f88c6b309..2600ba5b89f2a 100644 --- a/prelude/apple/tools/selective_debugging/scrubber.py +++ b/prelude/apple/tools/selective_debugging/scrubber.py @@ -55,7 +55,7 @@ def load_focused_targets_output_paths(json_file_path: str) -> Set[str]: # Visible for testing def _get_target_output_path_from_debug_file_path( debug_target_path: str, -): +) -> str: # This function assumes the debug file path created by buck2 is in the following format: # buck-out/isolation_dir/gen/project_cell/{hash}/.../__name__/libFoo.a parts = debug_target_path.split("/") diff --git a/prelude/apple/tools/selective_debugging/spec.py b/prelude/apple/tools/selective_debugging/spec.py index 250a2fa4303d9..6bba9c3b803b2 100644 --- a/prelude/apple/tools/selective_debugging/spec.py +++ b/prelude/apple/tools/selective_debugging/spec.py @@ -46,11 +46,11 @@ class Spec: include_build_target_patterns: List[BuildTargetPatternOutputPathMatcher] = field( init=False ) - include_regular_expressions: List[re.Pattern] = field(init=False) + include_regular_expressions: List[re.Pattern[str]] = field(init=False) exclude_build_target_patterns: List[BuildTargetPatternOutputPathMatcher] = field( init=False ) - exclude_regular_expressions: List[re.Pattern] = field(init=False) + exclude_regular_expressions: List[re.Pattern[str]] = field(init=False) def __post_init__(self) -> None: with open(self.spec_path, "r") as f: @@ -95,7 +95,7 @@ def scrub_debug_file_path(self, debug_file_path: str) -> bool: def _path_matches_pattern_or_expression( debug_file_path: str, patterns: List[BuildTargetPatternOutputPathMatcher], - expressions: List[re.Pattern], + expressions: List[re.Pattern[str]], ) -> bool: for pattern in patterns: if pattern.match_path(debug_file_path): diff --git a/prelude/apple/tools/split_arch_combine_dsym_bundles_tool.py b/prelude/apple/tools/split_arch_combine_dsym_bundles_tool.py index b7e99d8531d8f..77ec8a7c8ca3d 100644 --- a/prelude/apple/tools/split_arch_combine_dsym_bundles_tool.py +++ b/prelude/apple/tools/split_arch_combine_dsym_bundles_tool.py @@ -31,7 +31,7 @@ def _args_parser() -> argparse.ArgumentParser: return parser -def _main(): +def _main() -> None: args = _args_parser().parse_args() output_dwarf_path = os.path.join(args.output, "Contents/Resources/DWARF") diff --git a/prelude/apple/user/apple_resource_bundle.bzl b/prelude/apple/user/apple_resource_bundle.bzl index 18b82ec23bf1b..66c902dfb3887 100644 --- a/prelude/apple/user/apple_resource_bundle.bzl +++ b/prelude/apple/user/apple_resource_bundle.bzl @@ -37,6 +37,7 @@ def _apple_resource_bundle_attrs(): "info_plist": attrs.source(), "info_plist_substitutions": attrs.dict(key = attrs.string(), value = attrs.string(), sorted = False, default = {}), "labels": attrs.list(attrs.string(), default = []), + "privacy_manifest": attrs.option(attrs.source(), default = None), "product_name": attrs.option(attrs.string(), default = None), "resource_group": attrs.option(attrs.string(), default = None), "resource_group_map": resource_group_map_attr(), diff --git a/prelude/apple/user/apple_toolchain_override.bzl b/prelude/apple/user/apple_toolchain_override.bzl index 0a3886e49f3f5..8cce54845c581 100644 --- a/prelude/apple/user/apple_toolchain_override.bzl +++ b/prelude/apple/user/apple_toolchain_override.bzl @@ -31,6 +31,7 @@ def _impl(ctx: AnalysisContext) -> list[Provider]: lipo = base.lipo, min_version = base.min_version, momc = base.momc, + objdump = base.objdump, odrcov = base.odrcov, platform_path = base.platform_path, sdk_build_version = base.sdk_build_version, diff --git a/prelude/apple/apple_buck2_compatibility.bzl b/prelude/buck2_compatibility.bzl similarity index 94% rename from prelude/apple/apple_buck2_compatibility.bzl rename to prelude/buck2_compatibility.bzl index 09d870c0f7d17..439b344af51c4 100644 --- a/prelude/apple/apple_buck2_compatibility.bzl +++ b/prelude/buck2_compatibility.bzl @@ -15,6 +15,6 @@ Buck2Compatibility = enum( BUCK2_COMPATIBILITY_ATTRIB_NAME = "buck2_compatibility" BUCK2_COMPATIBILITY_ATTRIB_TYPE = attrs.enum(Buck2Compatibility.values(), default = "unknown") -def apple_check_buck2_compatibility(ctx: AnalysisContext): +def check_buck2_compatibility(ctx: AnalysisContext): if hasattr(ctx.attrs, "buck2_compatibility") and ctx.attrs.buck2_compatibility == "incompatible": warning("The target '{}' is marked as incompatible with buck2, output might be incorrect".format(ctx.label)) diff --git a/prelude/configurations/rules.bzl b/prelude/configurations/rules.bzl index 323560833d9d9..66536b8333b66 100644 --- a/prelude/configurations/rules.bzl +++ b/prelude/configurations/rules.bzl @@ -67,6 +67,9 @@ def platform_impl(ctx): ), ] +def configuration_alias_impl(ctx: AnalysisContext) -> list[Provider]: + return ctx.attrs.actual.providers + # TODO(cjhopman): Update the attributes for these ruletypes to declare the types of providers that they expect in their references. extra_attributes = { "platform": { @@ -76,6 +79,7 @@ extra_attributes = { implemented_rules = { "config_setting": config_setting_impl, + "configuration_alias": configuration_alias_impl, "constraint_setting": constraint_setting_impl, "constraint_value": constraint_value_impl, "platform": platform_impl, diff --git a/prelude/cxx/compile.bzl b/prelude/cxx/compile.bzl index 025f0991904e8..7941d522bd973 100644 --- a/prelude/cxx/compile.bzl +++ b/prelude/cxx/compile.bzl @@ -360,7 +360,8 @@ def create_compile_cmds( def compile_cxx( ctx: AnalysisContext, src_compile_cmds: list[CxxSrcCompileCommand], - pic: bool = False) -> list[CxxCompileOutput]: + pic: bool = False, + allow_cache_upload: bool = False) -> list[CxxCompileOutput]: """ For a given list of src_compile_cmds, generate output artifacts. """ @@ -461,6 +462,7 @@ def compile_cxx( category = src_compile_cmd.cxx_compile_cmd.category, identifier = identifier, dep_files = action_dep_files, + allow_cache_upload = allow_cache_upload, ) # If we're building with split debugging, where the debug info is in the @@ -557,6 +559,8 @@ def _get_compile_base(compiler_info: typing.Any) -> cmd_args: def _dep_file_type(ext: CxxExtension) -> [DepFileType, None]: # Raw assembly doesn't make sense to capture dep files for. + # .S is preprocessed assembly, but some builds use it with + # assemblers that don't support -MF, so leave depfiles off. if ext.value in (".s", ".S", ".asm"): return None elif ext.value == ".hip": diff --git a/prelude/cxx/cxx.bzl b/prelude/cxx/cxx.bzl index 54f13960c2e5f..535fa544c9bfe 100644 --- a/prelude/cxx/cxx.bzl +++ b/prelude/cxx/cxx.bzl @@ -25,6 +25,7 @@ load( "@prelude//linking:link_info.bzl", "Archive", "ArchiveLinkable", + "CxxSanitizerRuntimeInfo", "LibOutputStyle", "LinkArgs", "LinkCommandDebugOutputInfo", @@ -243,6 +244,8 @@ def cxx_binary_impl(ctx: AnalysisContext) -> list[Provider]: extra_providers = [] if output.link_command_debug_output: extra_providers.append(LinkCommandDebugOutputInfo(debug_outputs = [output.link_command_debug_output])) + if output.sanitizer_runtime_dir: + extra_providers.append(CxxSanitizerRuntimeInfo(runtime_dir = output.sanitizer_runtime_dir)) # When an executable is the output of a build, also materialize all the # unpacked external debuginfo that goes with it. This makes `buck2 build @@ -675,9 +678,14 @@ def cxx_test_impl(ctx: AnalysisContext) -> list[Provider]: use_project_relative_paths = re_executor != None, ), ) + [ - DefaultInfo(default_output = output.binary, other_outputs = output.runtime_files, sub_targets = output.sub_targets), + DefaultInfo( + default_output = output.binary, + other_outputs = output.runtime_files + output.external_debug_info_artifacts, + sub_targets = output.sub_targets, + ), output.compilation_db, output.xcode_data, + output.dist_info, ] def _get_params_for_android_binary_cxx_library() -> (CxxRuleSubTargetParams, CxxRuleProviderParams): diff --git a/prelude/cxx/cxx_executable.bzl b/prelude/cxx/cxx_executable.bzl index bf99f13464eed..27393f32bebbe 100644 --- a/prelude/cxx/cxx_executable.bzl +++ b/prelude/cxx/cxx_executable.bzl @@ -184,6 +184,7 @@ CxxExecutableOutput = record( linker_map_data = [CxxLinkerMapData, None], link_command_debug_output = field([LinkCommandDebugOutput, None], None), dist_info = DistInfo, + sanitizer_runtime_dir = field([Artifact, None], None), ) def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, is_cxx_test: bool = False) -> CxxExecutableOutput: @@ -325,6 +326,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, other_roots = link_group_extra_link_roots, prefer_stripped_objects = impl_params.prefer_stripped_objects, anonymous = ctx.attrs.anonymous_link_groups, + allow_cache_upload = impl_params.exe_allow_cache_upload, ) for name, linked_link_group in linked_link_groups.libs.items(): auto_link_groups[name] = linked_link_group.artifact @@ -678,6 +680,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, shared_libs = shlib_info.set, nondebug_runtime_files = runtime_files, ), + sanitizer_runtime_dir = link_result.sanitizer_runtime_dir, ) _CxxLinkExecutableResult = record( @@ -694,6 +697,7 @@ _CxxLinkExecutableResult = record( # Optional shared libs symlink tree symlinked_dir action shared_libs_symlink_tree = [list[Artifact], Artifact, None], linker_map_data = [CxxLinkerMapData, None], + sanitizer_runtime_dir = [Artifact, None], ) def _link_into_executable( @@ -709,7 +713,7 @@ def _link_into_executable( output_name = "{}{}".format(executable_name if executable_name else get_cxx_executable_product_name(ctx), "." + binary_extension if binary_extension else "") output = ctx.actions.declare_output(output_name) executable_args = executable_shared_lib_arguments( - ctx.actions, + ctx, get_cxx_toolchain_info(ctx), output, shared_libs, @@ -731,6 +735,7 @@ def _link_into_executable( external_debug_info = executable_args.external_debug_info, shared_libs_symlink_tree = executable_args.shared_libs_symlink_tree, linker_map_data = link_result.linker_map_data, + sanitizer_runtime_dir = executable_args.sanitizer_runtime_dir, ) def get_cxx_executable_product_name(ctx: AnalysisContext) -> str: diff --git a/prelude/cxx/cxx_library.bzl b/prelude/cxx/cxx_library.bzl index 714aaf1b30893..0b1f3fdf29b49 100644 --- a/prelude/cxx/cxx_library.bzl +++ b/prelude/cxx/cxx_library.bzl @@ -906,12 +906,12 @@ def cxx_compile_srcs( ) # Define object files. - pic_cxx_outs = compile_cxx(ctx, compile_cmd_output.src_compile_cmds, pic = True) + pic_cxx_outs = compile_cxx(ctx, compile_cmd_output.src_compile_cmds, pic = True, allow_cache_upload = ctx.attrs.allow_cache_upload) pic = _get_library_compile_output(ctx, pic_cxx_outs, impl_params.extra_link_input) non_pic = None if preferred_linkage != Linkage("shared"): - non_pic_cxx_outs = compile_cxx(ctx, compile_cmd_output.src_compile_cmds, pic = False) + non_pic_cxx_outs = compile_cxx(ctx, compile_cmd_output.src_compile_cmds, pic = False, allow_cache_upload = ctx.attrs.allow_cache_upload) non_pic = _get_library_compile_output(ctx, non_pic_cxx_outs, impl_params.extra_link_input) return _CxxCompiledSourcesOutput( diff --git a/prelude/cxx/cxx_link_utility.bzl b/prelude/cxx/cxx_link_utility.bzl index 77ab0424d1362..e6e446a67f8c6 100644 --- a/prelude/cxx/cxx_link_utility.bzl +++ b/prelude/cxx/cxx_link_utility.bzl @@ -155,10 +155,54 @@ ExecutableSharedLibArguments = record( external_debug_info = field(list[TransitiveSetArgsProjection], []), # Optional shared libs symlink tree symlinked_dir action. shared_libs_symlink_tree = field(list[Artifact] | Artifact | None, None), + # A directory containing sanitizer runtime shared libraries + sanitizer_runtime_dir = field(Artifact | None, None), ) +CxxSanitizerRuntimeArguments = record( + extra_link_args = field(list[ArgLike], []), + sanitizer_runtime_dir = field(Artifact | None, None), +) + +# @executable_path/Frameworks + +def _sanitizer_runtime_arguments( + ctx: AnalysisContext, + cxx_toolchain: CxxToolchainInfo, + output: Artifact) -> CxxSanitizerRuntimeArguments: + linker_info = cxx_toolchain.linker_info + target_sanitizer_runtime_enabled = ctx.attrs.sanitizer_runtime_enabled if hasattr(ctx.attrs, "sanitizer_runtime_enabled") else None + sanitizer_runtime_enabled = target_sanitizer_runtime_enabled if target_sanitizer_runtime_enabled != None else linker_info.sanitizer_runtime_enabled + if not sanitizer_runtime_enabled: + return CxxSanitizerRuntimeArguments() + + if linker_info.sanitizer_runtime_dir == None: + fail("C++ sanitizer runtime enabled but there's no runtime directory") + + if linker_info.type == "darwin": + runtime_rpath = cmd_args(linker_info.sanitizer_runtime_dir, format = "-Wl,-rpath,@executable_path/{}").relative_to(output, parent = 1) + + # Ignore_artifacts() as the runtime directory is not required at _link_ time + runtime_rpath = runtime_rpath.ignore_artifacts() + return CxxSanitizerRuntimeArguments( + extra_link_args = [ + runtime_rpath, + # Add rpaths in case the binary gets bundled and the app bundle is expected to be standalone. + # Not all transitive callers have `CxxPlatformInfo`, so just add both iOS and macOS rpaths. + # There's no downsides to having both, except dyld would check in both locations (and it won't + # find anything for the non-current platform). + "-Wl,-rpath,@loader_path/Frameworks", # iOS + "-Wl,-rpath,@executable_path/Frameworks", # iOS + "-Wl,-rpath,@loader_path/../Frameworks", # macOS + "-Wl,-rpath,@executable_path/../Frameworks", # macOS + ], + sanitizer_runtime_dir = linker_info.sanitizer_runtime_dir, + ) + + return CxxSanitizerRuntimeArguments() + def executable_shared_lib_arguments( - actions: AnalysisActions, + ctx: AnalysisContext, cxx_toolchain: CxxToolchainInfo, output: Artifact, shared_libs: dict[str, LinkedObject]) -> ExecutableSharedLibArguments: @@ -169,7 +213,7 @@ def executable_shared_lib_arguments( # External debug info is materialized only when the executable is the output # of a build. Do not add to runtime_files. external_debug_info = project_artifacts( - actions = actions, + actions = ctx.actions, tsets = [shlib.external_debug_info for shlib in shared_libs.values()], ) @@ -177,7 +221,7 @@ def executable_shared_lib_arguments( if len(shared_libs) > 0: if linker_type == "windows": - shared_libs_symlink_tree = [actions.symlink_file( + shared_libs_symlink_tree = [ctx.actions.symlink_file( shlib.output.basename, shlib.output, ) for _, shlib in shared_libs.items()] @@ -185,7 +229,7 @@ def executable_shared_lib_arguments( # Windows doesn't support rpath. else: - shared_libs_symlink_tree = actions.symlinked_dir( + shared_libs_symlink_tree = ctx.actions.symlinked_dir( shared_libs_symlink_tree_name(output), {name: shlib.output for name, shlib in shared_libs.items()}, ) @@ -196,11 +240,17 @@ def executable_shared_lib_arguments( rpath_arg = cmd_args(shared_libs_symlink_tree, format = "-Wl,-rpath,{}/{{}}".format(rpath_reference)).relative_to(output, parent = 1).ignore_artifacts() extra_link_args.append(rpath_arg) + sanitizer_runtime_args = _sanitizer_runtime_arguments(ctx, cxx_toolchain, output) + extra_link_args += sanitizer_runtime_args.extra_link_args + if sanitizer_runtime_args.sanitizer_runtime_dir != None: + runtime_files.append(sanitizer_runtime_args.sanitizer_runtime_dir) + return ExecutableSharedLibArguments( extra_link_args = extra_link_args, runtime_files = runtime_files, external_debug_info = external_debug_info, shared_libs_symlink_tree = shared_libs_symlink_tree, + sanitizer_runtime_dir = sanitizer_runtime_args.sanitizer_runtime_dir, ) def cxx_link_cmd_parts(toolchain: CxxToolchainInfo) -> ((RunInfo | cmd_args), cmd_args): diff --git a/prelude/cxx/cxx_toolchain.bzl b/prelude/cxx/cxx_toolchain.bzl index 3c9f3fa017f82..f71205f562db9 100644 --- a/prelude/cxx/cxx_toolchain.bzl +++ b/prelude/cxx/cxx_toolchain.bzl @@ -12,7 +12,7 @@ load("@prelude//cxx:headers.bzl", "HeaderMode", "HeadersAsRawHeadersMode") load("@prelude//cxx:linker.bzl", "LINKERS", "is_pdb_generated") load("@prelude//linking:link_info.bzl", "LinkOrdering", "LinkStyle") load("@prelude//linking:lto.bzl", "LtoMode", "lto_compiler_flags") -load("@prelude//utils:utils.bzl", "value_or") +load("@prelude//utils:utils.bzl", "flatten", "value_or") load("@prelude//decls/cxx_rules.bzl", "cxx_rules") def cxx_toolchain_impl(ctx): @@ -95,6 +95,9 @@ def cxx_toolchain_impl(ctx): independent_shlib_interface_linker_flags = ctx.attrs.shared_library_interface_flags, requires_archives = value_or(ctx.attrs.requires_archives, True), requires_objects = value_or(ctx.attrs.requires_objects, False), + sanitizer_runtime_dir = ctx.attrs.sanitizer_runtime_dir[DefaultInfo].default_outputs[0] if ctx.attrs.sanitizer_runtime_dir else None, + sanitizer_runtime_enabled = ctx.attrs.sanitizer_runtime_enabled, + sanitizer_runtime_files = flatten([runtime_file[DefaultInfo].default_outputs for runtime_file in ctx.attrs.sanitizer_runtime_files]), supports_distributed_thinlto = ctx.attrs.supports_distributed_thinlto, shared_dep_runtime_ld_flags = ctx.attrs.shared_dep_runtime_ld_flags, shared_library_name_default_prefix = _get_shared_library_name_default_prefix(ctx), @@ -194,6 +197,9 @@ def cxx_toolchain_extra_attributes(is_toolchain_rule): "public_headers_symlinks_enabled": attrs.bool(default = True), "ranlib": attrs.option(dep_type(providers = [RunInfo]), default = None), "requires_objects": attrs.bool(default = False), + "sanitizer_runtime_dir": attrs.option(attrs.dep(), default = None), # Use `attrs.dep()` as it's not a tool, always propagate target platform + "sanitizer_runtime_enabled": attrs.bool(default = False), + "sanitizer_runtime_files": attrs.set(attrs.dep(), sorted = True, default = []), # Use `attrs.dep()` as it's not a tool, always propagate target platform "shared_library_interface_mode": attrs.enum(ShlibInterfacesMode.values(), default = "disabled"), "shared_library_interface_producer": attrs.option(dep_type(providers = [RunInfo]), default = None), "split_debug_mode": attrs.enum(SplitDebugMode.values(), default = "none"), diff --git a/prelude/cxx/cxx_toolchain_types.bzl b/prelude/cxx/cxx_toolchain_types.bzl index 932179e3404e2..2a8a9b6d3e78b 100644 --- a/prelude/cxx/cxx_toolchain_types.bzl +++ b/prelude/cxx/cxx_toolchain_types.bzl @@ -7,12 +7,6 @@ load("@prelude//cxx:debug.bzl", "SplitDebugMode") -# For cases where our `ld` dependency provides more than an executable and -# would like to give us flags too. We use this to place the flags in the proper -# field (linker_flags), so that things that want ldflags without the linker -# executable can access those. -RichLinkerRunInfo = provider(fields = {"exe": provider_field(typing.Any, default = None), "flags": provider_field(typing.Any, default = None)}) - LinkerType = ["gnu", "darwin", "windows", "wasm"] ShlibInterfacesMode = enum("disabled", "enabled", "defined_only") @@ -49,6 +43,9 @@ LinkerInfo = provider( "mk_shlib_intf": provider_field(typing.Any, default = None), # "o" on Unix, "obj" on Windows "object_file_extension": provider_field(typing.Any, default = None), # str + "sanitizer_runtime_enabled": provider_field(bool, default = False), + "sanitizer_runtime_dir": provider_field([Artifact, None], default = None), + "sanitizer_runtime_files": provider_field(list[Artifact], default = []), "shlib_interfaces": provider_field(ShlibInterfacesMode), "shared_dep_runtime_ld_flags": provider_field(typing.Any, default = None), # "lib" on Linux/Mac/Android, "" on Windows. @@ -312,6 +309,7 @@ def cxx_toolchain_infos( "ldflags-shared": _shell_quote(linker_info.linker_flags), "ldflags-static": _shell_quote(linker_info.linker_flags), "ldflags-static-pic": _shell_quote(linker_info.linker_flags), + "objcopy": binary_utilities_info.objcopy, # TODO(T110378148): $(platform-name) is almost unusued. Should we remove it? "platform-name": platform_name, } diff --git a/prelude/cxx/dist_lto/README.md b/prelude/cxx/dist_lto/README.md index 1102134a2c296..88a4b80a75962 100644 --- a/prelude/cxx/dist_lto/README.md +++ b/prelude/cxx/dist_lto/README.md @@ -1,23 +1,27 @@ # Distributed ThinLTO in Buck2 + Sean Gillespie, April 2022 -This document is a technical overview into Buck2's implementation of a distributed ThinLTO. -Like all rules in Buck2, this implementation is written entirely in Starlark, contained in -`dist_lto.bzl` (in this same directory). +This document is a technical overview into Buck2's implementation of a +distributed ThinLTO. Like all rules in Buck2, this implementation is written +entirely in Starlark, contained in `dist_lto.bzl` (in this same directory). ## Motivation -First, I highly recommend watching [Teresa Johnson's CppCon2017 talk about ThinLTO](https://www.youtube.com/watch?v=p9nH2vZ2mNo), +First, I highly recommend watching +[Teresa Johnson's CppCon2017 talk about ThinLTO](https://www.youtube.com/watch?v=p9nH2vZ2mNo), which covers the topics in this section in much greater detail than I can. -C and C++ have long enjoyed significant optimizations at the hands of compilers. However, they have also -long suffered a fundamental limitation; a C or C++ compiler can only optimize code that it sees in a single -translation unit. For a language like C or C++, this means in practice that only code that is included via -the preprocessor or specified in the translation unit can be optimized as a single unit. C and C++ compilers -are unable to inline functions that are defined in different translation units. However, a crucial advantage -of this compilation model is that all C and C++ compiler invocations are *completely parallelizable*; despite -sacrificing some code quality, C and C++ compilation turns into a massively parallel problem with a serial -link step at the very end. +C and C++ have long enjoyed significant optimizations at the hands of compilers. +However, they have also long suffered a fundamental limitation; a C or C++ +compiler can only optimize code that it sees in a single translation unit. For a +language like C or C++, this means in practice that only code that is included +via the preprocessor or specified in the translation unit can be optimized as a +single unit. C and C++ compilers are unable to inline functions that are defined +in different translation units. However, a crucial advantage of this compilation +model is that all C and C++ compiler invocations are _completely +parallelizable_; despite sacrificing some code quality, C and C++ compilation +turns into a massively parallel problem with a serial link step at the very end. ``` flowchart LR; @@ -36,20 +40,25 @@ flowchart LR; c.o --> main; ``` -([Rendered](https://fburl.com/mermaid/rzup8o32). Compilation and optimization of a, b, and c can proceed in parallel.) - - -In cases where absolute performance is required, though, the inability to perform cross-translation-unit -(or "cross-module", in LLVM parlance) optimizations becomes more of a problem. To solve this, a new compilation -paradigm was designed, dubbed "Link-Time Optimization" (LTO). In this scheme, a compiler will not produce machine code -when processing a translation unit; rather, it will output the compiler's intermediate representation (e.g. LLVM bitcode). -Later on, when it is time for the linker to run, it will load all of the compiler IR into one giant module, run -optimization passes on the mega-module, and produce a final binary from that. - -This works quite well, if all that you're looking for is run-time performance. A major drawback of the LTO approach is -that all of the parallelism gained from optimizing translation units individually is now completely lost; instead, the -linker (using a plugin) will do a single-threaded pass of *all code* produced by compilation steps. This is extremely -slow, memory-intensive, and unable to be run incrementally. There are targets at Meta that simply can't be LTO-compiled +([Rendered](https://fburl.com/mermaid/rzup8o32). Compilation and optimization of +a, b, and c can proceed in parallel.) + +In cases where absolute performance is required, though, the inability to +perform cross-translation-unit (or "cross-module", in LLVM parlance) +optimizations becomes more of a problem. To solve this, a new compilation +paradigm was designed, dubbed "Link-Time Optimization" (LTO). In this scheme, a +compiler will not produce machine code when processing a translation unit; +rather, it will output the compiler's intermediate representation (e.g. LLVM +bitcode). Later on, when it is time for the linker to run, it will load all of +the compiler IR into one giant module, run optimization passes on the +mega-module, and produce a final binary from that. + +This works quite well, if all that you're looking for is run-time performance. A +major drawback of the LTO approach is that all of the parallelism gained from +optimizing translation units individually is now completely lost; instead, the +linker (using a plugin) will do a single-threaded pass of _all code_ produced by +compilation steps. This is extremely slow, memory-intensive, and unable to be +run incrementally. There are targets at Meta that simply can't be LTO-compiled because of their size. ``` @@ -74,15 +83,21 @@ flowchart LR; main.o --> |ld| main ``` -([Rendered](https://fburl.com/mermaid/kid35io9). `a.bc`, `b.bc`, and `c.bc` are LLVM bitcode; they are all merged -together into a single module, `a_b_c_optimized.bc`, which is then optimized and codegen'd into a final binary.) -The idea of ThinLTO comes from a desire to maintain the ability to optimize modules in parallel while still -allowing for profitable cross-module optimizations. The idea is this: +([Rendered](https://fburl.com/mermaid/kid35io9). `a.bc`, `b.bc`, and `c.bc` are +LLVM bitcode; they are all merged together into a single module, +`a_b_c_optimized.bc`, which is then optimized and codegen'd into a final +binary.) -1. Just like regular LTO, the compiler emits bitcode instead of machine code. However, it also contains some light -metadata such as a call graph of symbols within the module. -2. The monolithic LTO link is split into three steps: `index`, `opt`, and `link`. +The idea of ThinLTO comes from a desire to maintain the ability to optimize +modules in parallel while still allowing for profitable cross-module +optimizations. The idea is this: + +1. Just like regular LTO, the compiler emits bitcode instead of machine code. + However, it also contains some light metadata such as a call graph of symbols + within the module. +2. The monolithic LTO link is split into three steps: `index`, `opt`, and + `link`. ``` flowchart LR; @@ -117,137 +132,192 @@ flowchart LR; ([Rendered](https://fburl.com/mermaid/56oc99t5)) -The `index` step looks like a link step. However, it does not produce a final binary; instead, it looks at every -compiler IR input file that it receives and heuristically determines which other IR modules it should be optimized -with in order to achieve profitable optimizations. These modules might include functions that the index step thinks -probably will get inlined, or globals that are read in the target IR input file. The output of the index step is a -series of files on disk that indicate which sibling object files should be present when optimizing a particular object -file, for each object file in the linker command-line. - -The `opt` step runs in parallel for every object file. Each object file will be optimized using the compiler's -optimizer (e.g. `opt`, for LLVM). The optimizer will combine the objects that were referenced as part of the index -step as potentially profitable to include and optimize them all together. - -The `link` step takes the outputs of `opt` and links them together, like a normal linker. - -In practice, ThinLTO manages to recapture the inherent parallelism of C/C++ compilation by pushing the majority of work -to the parallel `opt` phase of execution. When LLVM performs ThinLTO by default, it will launch a thread pool and process -independent modules in parallel. ThinLTO does not produce as performant a binary as a monolithic LTO; however, in practice, -ThinLTO binaries [paired with AutoFDO](https://fburl.com/wiki/q480euco) perform comparably to monolithic LTO. Furthermore, -ThinLTO's greater efficiency allows for more expensive optimization passes to be run, which can further improve code quality +The `index` step looks like a link step. However, it does not produce a final +binary; instead, it looks at every compiler IR input file that it receives and +heuristically determines which other IR modules it should be optimized with in +order to achieve profitable optimizations. These modules might include functions +that the index step thinks probably will get inlined, or globals that are read +in the target IR input file. The output of the index step is a series of files +on disk that indicate which sibling object files should be present when +optimizing a particular object file, for each object file in the linker +command-line. + +The `opt` step runs in parallel for every object file. Each object file will be +optimized using the compiler's optimizer (e.g. `opt`, for LLVM). The optimizer +will combine the objects that were referenced as part of the index step as +potentially profitable to include and optimize them all together. + +The `link` step takes the outputs of `opt` and links them together, like a +normal linker. + +In practice, ThinLTO manages to recapture the inherent parallelism of C/C++ +compilation by pushing the majority of work to the parallel `opt` phase of +execution. When LLVM performs ThinLTO by default, it will launch a thread pool +and process independent modules in parallel. ThinLTO does not produce as +performant a binary as a monolithic LTO; however, in practice, ThinLTO binaries +[paired with AutoFDO](https://fburl.com/wiki/q480euco) perform comparably to +monolithic LTO. Furthermore, ThinLTO's greater efficiency allows for more +expensive optimization passes to be run, which can further improve code quality near that of a monolithic LTO. -This is all great, and ThinLTO has been in use at Meta for some time. However, Buck2 has the ability to take a step -further than Buck1 could ever have - Buck2 can distribute parallel `opt` actions across many machines via Remote Execution -to achieve drastic speedups in ThinLTO wall clock time, memory usage, and incrementality. +This is all great, and ThinLTO has been in use at Meta for some time. However, +Buck2 has the ability to take a step further than Buck1 could ever have - Buck2 +can distribute parallel `opt` actions across many machines via Remote Execution +to achieve drastic speedups in ThinLTO wall clock time, memory usage, and +incrementality. ## Buck2's Implementation -Buck2's role in a distributed ThinLTO compilation is to construct a graph of actions that directly mirrors the graph -that the `index` step outputs. The graph that the `index` step outputs is entirely dynamic and, as such, the build -system is only aware of what the graph could be after the `index` step is complete. Unlike Buck1 (or even Blaze/Bazel), -Buck2 has explicit support for this paradigm [("dynamic dependencies")](https://fburl.com/gdoc/zklwhkll). Therefore, for Buck2, the basic strategy looks like: - -1. Invoke `clang` to act as `index`. `index` will output a file for every object file that indicates what other modules -need to be present when running `opt` on the object file (an "imports file"). -2. Read imports files and construct a graph of dynamic `opt` actions whose dependencies mirror the contents of the imports files. -3. Collect the outputs from the `opt` actions and invoke the linker to produce a final binary. - -Action `2` is inherently dynamic, since it must read the contents of files produced as part of action `1`. Furthermore, -Buck2's support of `1` is complicated by the fact that certain Buck2 rules can produce an archive of object files as -an output (namely, the Rust compiler). As a result, Buck2's implementation of Distributed ThinLTO is highly dynamic. +Buck2's role in a distributed ThinLTO compilation is to construct a graph of +actions that directly mirrors the graph that the `index` step outputs. The graph +that the `index` step outputs is entirely dynamic and, as such, the build system +is only aware of what the graph could be after the `index` step is complete. +Unlike Buck1 (or even Blaze/Bazel), Buck2 has explicit support for this paradigm +[("dynamic dependencies")](https://fburl.com/gdoc/zklwhkll). Therefore, for +Buck2, the basic strategy looks like: + +1. Invoke `clang` to act as `index`. `index` will output a file for every object + file that indicates what other modules need to be present when running `opt` + on the object file (an "imports file"). +2. Read imports files and construct a graph of dynamic `opt` actions whose + dependencies mirror the contents of the imports files. +3. Collect the outputs from the `opt` actions and invoke the linker to produce a + final binary. + +Action `2` is inherently dynamic, since it must read the contents of files +produced as part of action `1`. Furthermore, Buck2's support of `1` is +complicated by the fact that certain Buck2 rules can produce an archive of +object files as an output (namely, the Rust compiler). As a result, Buck2's +implementation of Distributed ThinLTO is highly dynamic. Buck2's implementation contains four phases of actions: -1. `thin_lto_prepare`, which specifically handles archives containing LLVM IR and prepares them to be inputs to `thin_lto_index`, -2. `thin_lto_index`, which invokes LLVM's ThinLTO indexer to produce a imports list for every object file to be optimized, -3. `thin_lto_opt`, which optimizes each object file in parallel with its imports present, +1. `thin_lto_prepare`, which specifically handles archives containing LLVM IR + and prepares them to be inputs to `thin_lto_index`, +2. `thin_lto_index`, which invokes LLVM's ThinLTO indexer to produce a imports + list for every object file to be optimized, +3. `thin_lto_opt`, which optimizes each object file in parallel with its imports + present, 4. `thin_lto_link`, which links together the optimized code into a final binary. ### thin_lto_prepare -It is a reality of Buck2 today that some rules don't produce a statically-known list of object files. The list of object -files is known *a priori* during C/C++ compilation, since they have a one-to-one correspondence to source files; however, -the Rust compiler emits an archive of object files; without inspecting the archive, Buck2 has no way of knowing what -the contents of the archive are, or even if they contain bitcode at all. +It is a reality of Buck2 today that some rules don't produce a statically-known +list of object files. The list of object files is known _a priori_ during C/C++ +compilation, since they have a one-to-one correspondence to source files; +however, the Rust compiler emits an archive of object files; without inspecting +the archive, Buck2 has no way of knowing what the contents of the archive are, +or even if they contain bitcode at all. -Future steps (particularly `thin_lto_index`) are defined to only operate on a list of object files - a limitation [inherited from LLVM](https://lists.llvm.org/pipermail/llvm-dev/2019-June/133145.html). Therefore, it is the job of `thin_lto_prepare` to turn an archive into a list of objects - namely, by extracting the archive into a directory. +Future steps (particularly `thin_lto_index`) are defined to only operate on a +list of object files - a limitation +[inherited from LLVM](https://lists.llvm.org/pipermail/llvm-dev/2019-June/133145.html). +Therefore, it is the job of `thin_lto_prepare` to turn an archive into a list of +objects - namely, by extracting the archive into a directory. -Buck2 dispatches a `thin_lto_prepare` action for every archive. Each prepare action has two outputs: +Buck2 dispatches a `thin_lto_prepare` action for every archive. Each prepare +action has two outputs: -1. An **output directory** (called `objects` in the code), a directory that contains the unextracted contents of the archive. -2. A **archive manifest**, a JSON document containing a list of object files that are contained in the output directory. +1. An **output directory** (called `objects` in the code), a directory that + contains the unextracted contents of the archive. +2. A **archive manifest**, a JSON document containing a list of object files + that are contained in the output directory. -The core logic of this action is implemented in the Python script `dist_lto_prepare.py`, contained in the `tools` directory. In addition to unpacking each archive, Buck2 -keeps track of the list of archives as a Starlark array that will be referenced by index -in later steps. +The core logic of this action is implemented in the Python script +`dist_lto_prepare.py`, contained in the `tools` directory. In addition to +unpacking each archive, Buck2 keeps track of the list of archives as a Starlark +array that will be referenced by index in later steps. ### thin_lto_index -With all archives prepared, the next step is to invoke LLVM's ThinLTO indexer. For the purposes of Buck2, the indexer -looks like a linker; because of this, Buck2 must construct a reasonable link line. Buck2 does this by iterating over the -list of linkables that it has been given and constructing a link line from them. Uniquely for distributed ThinLTO, Buck2 -must wrap all objects that were derived from `thin_lto_prepare` (i.e. were extracted from archives) with `-Wl,--start-lib` -and `-Wl,--end-lib` to ensure that they are still treated as if they were archives by the indexer. - -Invoking the indexer is relatively straightforward in that Buck2 invokes it like it would any other linker. However, -once the indexer returns, Buck2 must post-process its output into a format that Buck2's Starlark can understand and -translate into a graph of dynamic `opt` actions. The first thing that Buck2 is write a "meta file" to disk, which -communicates inputs and outputs of `thin_lto_index` to a Python script, `dist_lto_planner.py`. The meta file contains -a list of 7-tuples, whose members are: - -1. The path to the source bitcode file. This is used as an index into - a dictionary that records much of the metadata coming - from these lines. +With all archives prepared, the next step is to invoke LLVM's ThinLTO indexer. +For the purposes of Buck2, the indexer looks like a linker; because of this, +Buck2 must construct a reasonable link line. Buck2 does this by iterating over +the list of linkables that it has been given and constructing a link line from +them. Uniquely for distributed ThinLTO, Buck2 must wrap all objects that were +derived from `thin_lto_prepare` (i.e. were extracted from archives) with +`-Wl,--start-lib` and `-Wl,--end-lib` to ensure that they are still treated as +if they were archives by the indexer. + +Invoking the indexer is relatively straightforward in that Buck2 invokes it like +it would any other linker. However, once the indexer returns, Buck2 must +post-process its output into a format that Buck2's Starlark can understand and +translate into a graph of dynamic `opt` actions. The first thing that Buck2 is +write a "meta file" to disk, which communicates inputs and outputs of +`thin_lto_index` to a Python script, `dist_lto_planner.py`. The meta file +contains a list of 7-tuples, whose members are: + +1. The path to the source bitcode file. This is used as an index into a + dictionary that records much of the metadata coming from these lines. 2. The path to an output file. `dist_lto_planner.py`is expected to place a - ThinLTO index file at this location (suffixed `.thinlto.bc`). -3. The path to an output plan. This script is expected to place a link - plan here (a JSON document indicating which other object files this) - object file depends on, among other things. -4. If this object file came from an archive, the index of the archive in - the Starlark archives array. + ThinLTO index file at this location (suffixed `.thinlto.bc`). +3. The path to an output plan. This script is expected to place a link plan here + (a JSON document indicating which other object files this) object file + depends on, among other things. +4. If this object file came from an archive, the index of the archive in the + Starlark archives array. 5. If this object file came from an archive, the name of the archive. -6. If this object file came from an archive, the path to an output plan. - This script is expected to produce an archive link plan here (a JSON) - document similar to the object link plan, except containing link - information for every file in the archive from which this object - came. +6. If this object file came from an archive, the path to an output plan. This + script is expected to produce an archive link plan here (a JSON) document + similar to the object link plan, except containing link information for every + file in the archive from which this object came. 7. If this object file came from an archive, the indexes directory of that - archive. This script is expected to place all ThinLTO indexes derived - from object files originating from this archive in that directory. - -There are two indices that are derived from this meta file: the object -index (`mapping["index"]`) and the archive index (`mapping["archive_index"]`). -These indices are indices into Starlark arrays for all objects and archive -linkables, respectively. `dist_lto_planner.py` script does not inspect them; rather, -it is expected to communicate these indices back to Starlark by writing them to the + archive. This script is expected to place all ThinLTO indexes derived from + object files originating from this archive in that directory. + +There are two indices that are derived from this meta file: the object index +(`mapping["index"]`) and the archive index (`mapping["archive_index"]`). These +indices are indices into Starlark arrays for all objects and archive linkables, +respectively. `dist_lto_planner.py` script does not inspect them; rather, it is +expected to communicate these indices back to Starlark by writing them to the link plan. -`dist_lto_planner.py` reads the index and imports file produced by LLVM and derives -a number of artifacts: - -1. For each object file, a `thinlto.bc` file (`bitcode_file`). This file is the same as the input bitcode file, except that LLVM has inserted a number of module imports to refer to the other modules that will be present when the object file is optimized. -2. For each object file, an optimization plan (`plan`). The optimization plan is a JSON document indicating how to construct an `opt` action for this object file. This plan includes -this object file's module imports, whether or not this file contains bitcode at all, a location to place the optimized object file, and a list of archives that this object file imported. -3. For each archive, an optimization plan (`archive_plan`), which contains optimization plans for all of the object files contained within the archive. - -This action is a dynamic action because, in the case that there are archives that needed to be preprocessed by `thin_lto_prepare`, this action must read the archive manifest. +`dist_lto_planner.py` reads the index and imports file produced by LLVM and +derives a number of artifacts: + +1. For each object file, a `thinlto.bc` file (`bitcode_file`). This file is the + same as the input bitcode file, except that LLVM has inserted a number of + module imports to refer to the other modules that will be present when the + object file is optimized. +2. For each object file, an optimization plan (`plan`). The optimization plan is + a JSON document indicating how to construct an `opt` action for this object + file. This plan includes this object file's module imports, whether or not + this file contains bitcode at all, a location to place the optimized object + file, and a list of archives that this object file imported. +3. For each archive, an optimization plan (`archive_plan`), which contains + optimization plans for all of the object files contained within the archive. + +This action is a dynamic action because, in the case that there are archives +that needed to be preprocessed by `thin_lto_prepare`, this action must read the +archive manifest. ### thin_lto_opt -After `thin_lto_index` completes, Buck2 launches `thin_lto_opt` actions for every object file and for every archive. For each object file, Buck2 reads that object file's optimization plan. -At this phase, it is Buck2's responsibility to declare dependencies on every object file referenced by that object's compilation plan; it does so here by adding `hidden` dependencies -on every object file and archive that the archive plan says that this object depends on. - -`thin_lto_opt` uses a Python wrapper around LLVM because of a bug (T116695431) where LTO fatal errors don't prevent `clang` from returning an exit code of zero. The Python script wraps -`clang` and exits with a non-zero exit code if `clang` produced an empty object file. - -For each archive, Buck2 reads the archive's optimization plan and constructs additional `thin_lto_opt` actions for each object file contained in the archive. Buck2 creates a directory of -symlinks (`opt_objects`) that either contains symlinks to optimized object files (if the object file contained bitcode) or the original object file (if it didn't). The purpose of this symlink directory is to allow the final link to consume object files directly -from this directory without having to know whether they were optimized or not. Paths to these files are passed to the link step -via the optimization manifest (`opt_manifest`). +After `thin_lto_index` completes, Buck2 launches `thin_lto_opt` actions for +every object file and for every archive. For each object file, Buck2 reads that +object file's optimization plan. At this phase, it is Buck2's responsibility to +declare dependencies on every object file referenced by that object's +compilation plan; it does so here by adding `hidden` dependencies on every +object file and archive that the archive plan says that this object depends on. + +`thin_lto_opt` uses a Python wrapper around LLVM because of a bug (T116695431) +where LTO fatal errors don't prevent `clang` from returning an exit code of +zero. The Python script wraps `clang` and exits with a non-zero exit code if +`clang` produced an empty object file. + +For each archive, Buck2 reads the archive's optimization plan and constructs +additional `thin_lto_opt` actions for each object file contained in the archive. +Buck2 creates a directory of symlinks (`opt_objects`) that either contains +symlinks to optimized object files (if the object file contained bitcode) or the +original object file (if it didn't). The purpose of this symlink directory is to +allow the final link to consume object files directly from this directory +without having to know whether they were optimized or not. Paths to these files +are passed to the link step via the optimization manifest (`opt_manifest`). ### thin_lto_link -The final link step. Similar to `thin_lto_index`, this involves creating a link line to feed to the linker that uses the optimized artifacts that we just calculated. In cases where Buck2 -would put an archive on the link line, it instead inserts `-Wl,--start-lib`, `-Wl,--end-lib`, and references to the objects in `opt_objects`. +The final link step. Similar to `thin_lto_index`, this involves creating a link +line to feed to the linker that uses the optimized artifacts that we just +calculated. In cases where Buck2 would put an archive on the link line, it +instead inserts `-Wl,--start-lib`, `-Wl,--end-lib`, and references to the +objects in `opt_objects`. diff --git a/prelude/cxx/dist_lto/tools/dist_lto_opt.py b/prelude/cxx/dist_lto/tools/dist_lto_opt.py index 183b24c54826f..bd8c7d4e43b85 100644 --- a/prelude/cxx/dist_lto/tools/dist_lto_opt.py +++ b/prelude/cxx/dist_lto/tools/dist_lto_opt.py @@ -46,8 +46,6 @@ def _filter_flags(clang_flags: List[str]) -> List[str]: # noqa: C901 # this setting matches current llvm implementation: # https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/LTO/Config.h#L57 "-O2", - # TODO(T139459170): Remove after clang-15. NPM is the default. - "-fexperimental-new-pass-manager", "-ffunction-sections", "-fdata-sections", ] diff --git a/prelude/cxx/dist_lto/tools/tests/test_dist_lto_opt.py b/prelude/cxx/dist_lto/tools/tests/test_dist_lto_opt.py index 83cda1e6d94e1..454690ab6ed7f 100644 --- a/prelude/cxx/dist_lto/tools/tests/test_dist_lto_opt.py +++ b/prelude/cxx/dist_lto/tools/tests/test_dist_lto_opt.py @@ -30,7 +30,6 @@ def test_filter_flags(self): flags, [ "-O2", - "-fexperimental-new-pass-manager", "-ffunction-sections", "-fdata-sections", "-mllvm", @@ -92,7 +91,6 @@ def test_filter_flags_hhvm_case_rev_0f8618f31(self): "-Wl,-mllvm,-hot-callsite-threshold=12000", "-Wl,--lto-whole-program-visibility", "-fwhole-program-vtables", - "-fexperimental-new-pass-manager", "-Wl,--no-discard-section=.nv_fatbin", "-Wl,--no-discard-section=.nvFatBinSegment", "fbcode/tools/build/move_gpu_sections_implicit_linker_script.txt", @@ -143,7 +141,6 @@ def test_filter_flags_hhvm_case_rev_0f8618f31(self): flags, [ "-O2", - "-fexperimental-new-pass-manager", "-ffunction-sections", "-fdata-sections", "-mllvm", @@ -190,7 +187,6 @@ def test_filter_flags_unicorn_case_rev_0f8618f31(self): "-Wl,--discard-section=.rela.debug_types", "-Wl,-O1", "-Wl,--build-id=sha1", - "-fexperimental-new-pass-manager", "-Xlinker", "-znow", "-Xlinker", @@ -262,7 +258,6 @@ def test_filter_flags_unicorn_case_rev_0f8618f31(self): flags, [ "-O2", - "-fexperimental-new-pass-manager", "-ffunction-sections", "-fdata-sections", "-fprofile-sample-use=buck-out/v2/gen/fbcode/40fc99293b37c503/fdo/autofdo/default_profile/__autofdo__/out/profile", diff --git a/prelude/cxx/groups.bzl b/prelude/cxx/groups.bzl index 8187869919453..1237e783a04be 100644 --- a/prelude/cxx/groups.bzl +++ b/prelude/cxx/groups.bzl @@ -44,6 +44,8 @@ FilterType = enum( "label", # Filters for targets for the build target pattern defined after "pattern:". "pattern", + # Filters for targets matching the regex pattern defined after "target_regex:". + "target_regex", ) BuildTargetFilter = record( @@ -56,6 +58,11 @@ LabelFilter = record( _type = field(FilterType, FilterType("label")), ) +TargetRegexFilter = record( + regex = regex, + _type = field(FilterType, FilterType("target_regex")), +) + # Label for special group mapping which makes every target associated with it to be included in all groups MATCH_ALL_LABEL = "MATCH_ALL" @@ -70,7 +77,7 @@ GroupMapping = record( # The type of traversal to use. traversal = field(Traversal, Traversal("tree")), # Optional filter type to apply to the traversal. - filters = field(list[[BuildTargetFilter, LabelFilter]], []), + filters = field(list[[BuildTargetFilter, LabelFilter, TargetRegexFilter]], []), # Preferred linkage for this target when added to a link group. preferred_linkage = field([Linkage, None], None), ) @@ -191,7 +198,7 @@ def _parse_traversal_from_mapping(entry: str) -> Traversal: else: fail("Unrecognized group traversal type: " + entry) -def _parse_filter(entry: str) -> [BuildTargetFilter, LabelFilter]: +def _parse_filter(entry: str) -> [BuildTargetFilter, LabelFilter, TargetRegexFilter]: for prefix in ("label:", "tag:"): label_regex = strip_prefix(prefix, entry) if label_regex != None: @@ -203,15 +210,19 @@ def _parse_filter(entry: str) -> [BuildTargetFilter, LabelFilter]: regex = regex("^{}$".format(label_regex), fancy = True), ) + target_regex = strip_prefix("target_regex:", entry) + if target_regex != None: + return TargetRegexFilter(regex = regex("^{}$".format(target_regex), fancy = True)) + pattern = strip_prefix("pattern:", entry) if pattern != None: return BuildTargetFilter( pattern = parse_build_target_pattern(pattern), ) - fail("Invalid group mapping filter: {}\nFilter must begin with `label:`, `tag:`, or `pattern:`.".format(entry)) + fail("Invalid group mapping filter: {}\nFilter must begin with `label:`, `tag:`, `target_regex` or `pattern:`.".format(entry)) -def _parse_filter_from_mapping(entry: [list[str], str, None]) -> list[[BuildTargetFilter, LabelFilter]]: +def _parse_filter_from_mapping(entry: [list[str], str, None]) -> list[[BuildTargetFilter, LabelFilter, TargetRegexFilter]]: if type(entry) == type([]): return [_parse_filter(e) for e in entry] if type(entry) == type(""): @@ -266,6 +277,9 @@ def _find_targets_in_mapping( if filter._type == FilterType("label"): if not any_labels_match(filter.regex, labels): return False + elif filter._type == FilterType("target_regex"): + target_str = str(target.raw_target()) + return filter.regex.match(target_str) elif not filter.pattern.matches(target): return False return True diff --git a/prelude/cxx/headers.bzl b/prelude/cxx/headers.bzl index 150920d2d3f99..d41dc1f295c31 100644 --- a/prelude/cxx/headers.bzl +++ b/prelude/cxx/headers.bzl @@ -358,5 +358,5 @@ def _mk_hmap(ctx: AnalysisContext, name: str, headers: dict[str, (Artifact, str) cmd.add(["--mappings-file", hmap_args_file]).hidden(header_args) if project_root_file: cmd.add(["--project-root-file", project_root_file]) - ctx.actions.run(cmd, category = "generate_hmap", identifier = name) + ctx.actions.run(cmd, category = "generate_hmap", identifier = name, allow_cache_upload = ctx.attrs.allow_cache_upload) return output diff --git a/prelude/cxx/link_groups.bzl b/prelude/cxx/link_groups.bzl index fa353e20a69fa..8bd273b64ea25 100644 --- a/prelude/cxx/link_groups.bzl +++ b/prelude/cxx/link_groups.bzl @@ -419,7 +419,9 @@ def get_filtered_labels_to_links_map( target_link_group = link_group_mappings.get(target) # Always add force-static libs to the link. - if force_static_follows_dependents and node.preferred_linkage == Linkage("static"): + if (force_static_follows_dependents and + node.preferred_linkage == Linkage("static") and + not node.ignore_force_static_follows_dependents): add_link(target, output_style) elif not target_link_group and not link_group: # Ungrouped linkable targets belong to the unlabeled executable @@ -535,7 +537,7 @@ def find_relevant_roots( # link group. def collect_and_traverse_roots(roots, node_target): node = linkable_graph_node_map.get(node_target) - if node.preferred_linkage == Linkage("static"): + if node.preferred_linkage == Linkage("static") and not node.ignore_force_static_follows_dependents: return node.deps + node.exported_deps node_link_group = link_group_mappings.get(node_target) if node_link_group == MATCH_ALL_LABEL: @@ -573,7 +575,8 @@ def _create_link_group( link_group_libs: dict[str, ([Label, None], LinkInfos)] = {}, prefer_stripped_objects: bool = False, category_suffix: [str, None] = None, - anonymous: bool = False) -> [LinkedObject, None]: + anonymous: bool = False, + allow_cache_upload = False) -> [LinkedObject, None]: """ Link a link group library, described by a `LinkGroupLibSpec`. This is intended to handle regular shared libs and e.g. Python extensions. @@ -662,6 +665,7 @@ def _create_link_group( # TODO: anonymous targets cannot be used with dynamic output yet enable_distributed_thinlto = False if anonymous else spec.group.attrs.enable_distributed_thinlto, link_execution_preference = LinkExecutionPreference("any"), + allow_cache_upload = allow_cache_upload, ), anonymous = anonymous, ) @@ -782,7 +786,8 @@ def create_link_groups( linkable_graph_node_map: dict[Label, LinkableNode] = {}, link_group_preferred_linkage: dict[Label, Linkage] = {}, link_group_mappings: [dict[Label, str], None] = None, - anonymous: bool = False) -> _LinkedLinkGroups: + anonymous: bool = False, + allow_cache_upload = False) -> _LinkedLinkGroups: # Generate stubs first, so that subsequent links can link against them. link_group_shared_links = {} specs = [] @@ -840,6 +845,7 @@ def create_link_groups( prefer_stripped_objects = prefer_stripped_objects, category_suffix = "link_group", anonymous = anonymous, + allow_cache_upload = allow_cache_upload, ) if link_group_lib == None: diff --git a/prelude/cxx/omnibus.bzl b/prelude/cxx/omnibus.bzl index 801f921427499..b26c06f284786 100644 --- a/prelude/cxx/omnibus.bzl +++ b/prelude/cxx/omnibus.bzl @@ -153,7 +153,7 @@ def get_excluded(deps: list[Dependency] = []) -> dict[Label, None]: def create_linkable_root( link_infos: LinkInfos, name: [str, None] = None, - deps: list[Dependency] = []) -> LinkableRootInfo: + deps: list[LinkableGraph | Dependency] = []) -> LinkableRootInfo: # Only include dependencies that are linkable. return LinkableRootInfo( name = name, @@ -587,11 +587,18 @@ def _build_omnibus_spec( if label not in excluded } - # Find the deps of the root nodes. These form the roots of the nodes - # included in the omnibus link. + # Find the deps of the root nodes that should be linked into + # 'libomnibus.so'. + # + # If a dep indicates preferred linkage static, it is linked directly into + # this omnimbus root and therefore not added to `first_order_root_deps` and + # thereby will not be linked into 'libomnibus.so'. If the dep does not + # indicate preferred linkage static, then it is added to + # `first_order_root_deps` and thereby will be linked into 'libomnibus.so'. first_order_root_deps = [] for label in _link_deps(graph.nodes, flatten([r.deps for r in roots.values()]), get_cxx_toolchain_info(ctx).pic_behavior): - # We only consider deps which aren't *only* statically linked. + # Per the comment above, only consider deps which aren't *only* + # statically linked. if _is_static_only(graph.nodes[label]): continue diff --git a/prelude/cxx/user/cxx_toolchain_override.bzl b/prelude/cxx/user/cxx_toolchain_override.bzl index 1cc4cd7262902..6d4437f4b2dab 100644 --- a/prelude/cxx/user/cxx_toolchain_override.bzl +++ b/prelude/cxx/user/cxx_toolchain_override.bzl @@ -16,7 +16,7 @@ load( load("@prelude//linking:lto.bzl", "LtoMode") load("@prelude//user:rule_spec.bzl", "RuleRegistrationSpec") load("@prelude//utils:pick.bzl", _pick = "pick", _pick_and_add = "pick_and_add", _pick_bin = "pick_bin", _pick_dep = "pick_dep") -load("@prelude//utils:utils.bzl", "map_val", "value_or") +load("@prelude//utils:utils.bzl", "flatten", "map_val", "value_or") def _cxx_toolchain_override(ctx): base_toolchain = ctx.attrs.base[CxxToolchainInfo] @@ -75,6 +75,8 @@ def _cxx_toolchain_override(ctx): # linker flags should be changed as well. pdb_expected = linker_type == "windows" and pdb_expected shlib_interfaces = ShlibInterfacesMode(ctx.attrs.shared_library_interface_mode) if ctx.attrs.shared_library_interface_mode else None + sanitizer_runtime_dir = ctx.attrs.sanitizer_runtime_dir[DefaultInfo].default_outputs[0] if ctx.attrs.sanitizer_runtime_dir else None + sanitizer_runtime_files = flatten([runtime_file[DefaultInfo].default_outputs for runtime_file in ctx.attrs.sanitizer_runtime_files]) if ctx.attrs.sanitizer_runtime_files != None else None linker_info = LinkerInfo( archiver = _pick_bin(ctx.attrs.archiver, base_linker_info.archiver), archiver_type = base_linker_info.archiver_type, @@ -98,6 +100,9 @@ def _cxx_toolchain_override(ctx): requires_objects = base_linker_info.requires_objects, supports_distributed_thinlto = base_linker_info.supports_distributed_thinlto, independent_shlib_interface_linker_flags = base_linker_info.independent_shlib_interface_linker_flags, + sanitizer_runtime_dir = value_or(sanitizer_runtime_dir, base_linker_info.sanitizer_runtime_dir), + sanitizer_runtime_enabled = value_or(ctx.attrs.sanitizer_runtime_enabled, base_linker_info.sanitizer_runtime_enabled), + sanitizer_runtime_files = value_or(sanitizer_runtime_files, base_linker_info.sanitizer_runtime_files), shared_dep_runtime_ld_flags = [], shared_library_name_default_prefix = ctx.attrs.shared_library_name_default_prefix if ctx.attrs.shared_library_name_default_prefix != None else base_linker_info.shared_library_name_default_prefix, shared_library_name_format = ctx.attrs.shared_library_name_format if ctx.attrs.shared_library_name_format != None else base_linker_info.shared_library_name_format, @@ -206,6 +211,9 @@ def _cxx_toolchain_override_inheriting_target_platform_attrs(is_toolchain_rule): "platform_name": attrs.option(attrs.string(), default = None), "produce_interface_from_stub_shared_library": attrs.option(attrs.bool(), default = None), "ranlib": attrs.option(dep_type(providers = [RunInfo]), default = None), + "sanitizer_runtime_dir": attrs.option(attrs.dep(), default = None), # Use `attrs.dep()` as it's not a tool, always propagate target platform + "sanitizer_runtime_enabled": attrs.bool(default = False), + "sanitizer_runtime_files": attrs.option(attrs.set(attrs.dep(), sorted = True, default = []), default = None), # Use `attrs.dep()` as it's not a tool, always propagate target platform "shared_library_interface_mode": attrs.option(attrs.enum(ShlibInterfacesMode.values()), default = None), "shared_library_name_default_prefix": attrs.option(attrs.string(), default = None), "shared_library_name_format": attrs.option(attrs.string(), default = None), diff --git a/prelude/decls/android_rules.bzl b/prelude/decls/android_rules.bzl index 0adf61a993b8b..3c15a772a8cd0 100644 --- a/prelude/decls/android_rules.bzl +++ b/prelude/decls/android_rules.bzl @@ -116,6 +116,7 @@ android_aar = prelude_rule( "contacts": attrs.list(attrs.string(), default = []), "default_host_platform": attrs.option(attrs.configuration_label(), default = None), "enable_relinker": attrs.bool(default = False), + "excluded_java_deps": attrs.list(attrs.dep(), default = []), "extra_arguments": attrs.list(attrs.string(), default = []), "extra_kotlinc_arguments": attrs.list(attrs.string(), default = []), "extra_non_source_only_abi_kotlinc_arguments": attrs.list(attrs.string(), default = []), @@ -151,6 +152,7 @@ android_aar = prelude_rule( "srcs": attrs.list(attrs.source(), default = []), "target": attrs.option(attrs.string(), default = None), "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -765,6 +767,7 @@ android_library = prelude_rule( "runtime_deps": attrs.list(attrs.dep(), default = []), "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -915,6 +918,7 @@ android_prebuilt_aar = prelude_rule( "default_host_platform": attrs.option(attrs.configuration_label(), default = None), "deps": attrs.list(attrs.dep(), default = []), "desugar_deps": attrs.list(attrs.dep(), default = []), + "for_primary_apk": attrs.bool(default = False), "labels": attrs.list(attrs.string(), default = []), "licenses": attrs.list(attrs.source(), default = []), "maven_coords": attrs.option(attrs.string(), default = None), @@ -1447,9 +1451,10 @@ robolectric_test = prelude_rule( "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "use_cxx_libraries": attrs.option(attrs.bool(), default = None), "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), - "used_as_dependency_deprecated_do_not_use": attrs.bool(default = True), + "used_as_dependency_deprecated_do_not_use": attrs.bool(default = False), "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), "vm_args": attrs.list(attrs.arg(), default = []), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } | jvm_common.k2() | re_test_common.test_args() ), diff --git a/prelude/decls/apple_common.bzl b/prelude/decls/apple_common.bzl index 3c44e6c83e4df..92ff680364427 100644 --- a/prelude/decls/apple_common.bzl +++ b/prelude/decls/apple_common.bzl @@ -53,22 +53,20 @@ def _header_path_prefix_arg(): using ``` - apple_library( name = "Library", headers = glob(["**/*.h"]), header_path_prefix = "Lib", ) - ``` + can be imported using following mapping ``` - Library/SubDir/Header1.h -> Lib/Header1.h Library/Header2.h -> Lib/Header2.h - ``` + Defaults to the short name of the target. Can contain forward slashes (`/`), but cannot start with one. See `headers` for more information. """), @@ -128,6 +126,13 @@ def _extra_xcode_files(): """), } +def _privacy_manifest_arg(): + return { + "privacy_manifest": attrs.option(attrs.source(), default = None, doc = """ + A path to an `.xcprivacy` file that will be placed in the bundle. +"""), + } + apple_common = struct( headers_arg = _headers_arg, exported_headers_arg = _exported_headers_arg, @@ -138,4 +143,5 @@ apple_common = struct( info_plist_substitutions_arg = _info_plist_substitutions_arg, extra_xcode_sources = _extra_xcode_sources, extra_xcode_files = _extra_xcode_files, + privacy_manifest_arg = _privacy_manifest_arg, ) diff --git a/prelude/decls/core_rules.bzl b/prelude/decls/core_rules.bzl index 08e5528bc1690..2831ac2cf8d48 100644 --- a/prelude/decls/core_rules.bzl +++ b/prelude/decls/core_rules.bzl @@ -221,6 +221,23 @@ config_setting = prelude_rule( ), ) +configuration_alias = prelude_rule( + name = "configuration_alias", + docs = "", + examples = None, + further = None, + attrs = ( + # @unsorted-dict-items + { + # configuration_alias acts like alias but for configuration rules. + + # The configuration_alias itself is a configuration rule and the `actual` argument is + # expected to be a configuration rule as well. + "actual": attrs.dep(pulls_and_pushes_plugins = plugins.All), + } + ), +) + configured_alias = prelude_rule( name = "configured_alias", docs = "", @@ -1486,6 +1503,7 @@ core_rules = struct( alias = alias, command_alias = command_alias, config_setting = config_setting, + configuration_alias = configuration_alias, configured_alias = configured_alias, constraint_setting = constraint_setting, constraint_value = constraint_value, diff --git a/prelude/decls/cxx_rules.bzl b/prelude/decls/cxx_rules.bzl index a942522ab9f2b..8234ff2d60399 100644 --- a/prelude/decls/cxx_rules.bzl +++ b/prelude/decls/cxx_rules.bzl @@ -588,7 +588,8 @@ cxx_library = prelude_rule( "weak_framework_names": attrs.list(attrs.string(), default = []), "xcode_private_headers_symlinks": attrs.option(attrs.bool(), default = None), "xcode_public_headers_symlinks": attrs.option(attrs.bool(), default = None), - } + } | + buck.allow_cache_upload_arg() ), ) @@ -849,7 +850,8 @@ cxx_test = prelude_rule( "use_default_test_main": attrs.option(attrs.bool(), default = None), "version_universe": attrs.option(attrs.string(), default = None), "weak_framework_names": attrs.list(attrs.string(), default = []), - } + } | + buck.allow_cache_upload_arg() ), ) @@ -1093,7 +1095,8 @@ prebuilt_cxx_library = prelude_rule( "versioned_soname": attrs.option(attrs.versioned(attrs.string()), default = None), "versioned_static_lib": attrs.option(attrs.versioned(attrs.source()), default = None), "versioned_static_pic_lib": attrs.option(attrs.versioned(attrs.source()), default = None), - } + } | + buck.allow_cache_upload_arg() ), ) diff --git a/prelude/decls/erlang_rules.bzl b/prelude/decls/erlang_rules.bzl index 11171cd09998b..1c4b0ac5e3d18 100644 --- a/prelude/decls/erlang_rules.bzl +++ b/prelude/decls/erlang_rules.bzl @@ -121,6 +121,9 @@ rules_attributes = { [application_opt()](https://www.erlang.org/doc/man/application.html#load-2). The key-value pair will be stored in the applications `.app` file and can be accessed by `file:consult/1`. """), + "include_src": attrs.bool(default = True, doc = """ + This field controlls if the generated application directory contains a src/ directory with the Erlang code or not. + """), "includes": attrs.list(attrs.source(), default = [], doc = """ The public header files accessible via `-include_lib("appname/include/header.hrl")` from other erlang files. """), @@ -242,6 +245,10 @@ rules_attributes = { "extra_ct_hooks": attrs.list(attrs.string(), default = [], doc = """ List of additional Common Test hooks. The strings are interpreted as Erlang terms. """), + "extra_erl_flags": attrs.list(attrs.string(), default = [], doc = """ + List of additional command line arguments given to the erl command invocation. These + arguments are added to the front of the argument list. + """), "preamble": attrs.string(default = read_root_config("erlang", "erlang_test_preamble", "test:info(),test:ensure_initialized(),test:start_shell()."), doc = """ """), "property_tests": attrs.list(attrs.dep(), default = [], doc = """ diff --git a/prelude/decls/genrule_common.bzl b/prelude/decls/genrule_common.bzl index 0538c1ec469c6..1b84e280899e9 100644 --- a/prelude/decls/genrule_common.bzl +++ b/prelude/decls/genrule_common.bzl @@ -49,13 +49,13 @@ def _cmd_arg(): A string expansion of the `srcs` argument delimited by the `environment_expansion_separator` argument where each element of `srcs` will be translated - into an absolute path. + into a relative path. `${SRCDIR}` - The absolute path to a directory to which sources are copied + The relative path to a directory to which sources are copied prior to running the command. @@ -97,8 +97,7 @@ def _cmd_arg(): to be dependencies of the `genrule()`. - Note that the paths returned by these macros are *absolute* paths. You should convert these paths to be relative paths before - embedding them in, for example, a shell script or batch file. Using + Note that the paths returned by these macros are *relative* paths. Using relative paths ensures that your builds are *hermetic*, that is, they are reproducible across different machine environments. diff --git a/prelude/decls/go_common.bzl b/prelude/decls/go_common.bzl index 845f3861bc7ae..46e5305d4e878 100644 --- a/prelude/decls/go_common.bzl +++ b/prelude/decls/go_common.bzl @@ -124,6 +124,14 @@ def _embedcfg_arg(): """), } +def _cgo_enabled_arg(): + return { + "cgo_enabled": attrs.option(attrs.bool(), default = None, doc = """ + Experimental: Analog of CGO_ENABLED environment-variable. + None will be coverted to True if cxx_toolchain availabe for current configuration, otherwiese False. +"""), + } + go_common = struct( deps_arg = _deps_arg, srcs_arg = _srcs_arg, @@ -136,4 +144,5 @@ go_common = struct( linker_flags_arg = _linker_flags_arg, external_linker_flags_arg = _external_linker_flags_arg, embedcfg_arg = _embedcfg_arg, + cgo_enabled_arg = _cgo_enabled_arg, ) diff --git a/prelude/decls/go_rules.bzl b/prelude/decls/go_rules.bzl index 2ec5f5ebdd951..973c35bf36eec 100644 --- a/prelude/decls/go_rules.bzl +++ b/prelude/decls/go_rules.bzl @@ -14,6 +14,7 @@ load(":common.bzl", "CxxRuntimeType", "CxxSourceType", "HeadersAsRawHeadersMode" load(":cxx_common.bzl", "cxx_common") load(":go_common.bzl", "go_common") load(":native_common.bzl", "native_common") +load(":re_test_common.bzl", "re_test_common") BuildMode = ["executable", "c_shared", "c_archive"] @@ -125,7 +126,8 @@ cgo_library = prelude_rule( "thin_lto": attrs.bool(default = False), "version_universe": attrs.option(attrs.string(), default = None), "weak_framework_names": attrs.list(attrs.string(), default = []), - } + } | + buck.allow_cache_upload_arg() ), ) @@ -182,6 +184,7 @@ go_binary = prelude_rule( go_common.linker_flags_arg() | go_common.external_linker_flags_arg() | go_common.embedcfg_arg() | + go_common.cgo_enabled_arg() | { "resources": attrs.list(attrs.source(), default = [], doc = """ Static files to be symlinked into the working directory of the test. You can access these in your @@ -266,6 +269,7 @@ go_exported_library = prelude_rule( go_common.assembler_flags_arg() | go_common.linker_flags_arg() | go_common.external_linker_flags_arg() | + go_common.cgo_enabled_arg() | { "resources": attrs.list(attrs.source(), default = [], doc = """ Static files to be symlinked into the working directory of the test. You can access these in your @@ -415,6 +419,7 @@ go_test = prelude_rule( go_common.linker_flags_arg() | go_common.external_linker_flags_arg() | go_common.embedcfg_arg() | + go_common.cgo_enabled_arg() | { "resources": attrs.list(attrs.source(), default = [], doc = """ Static files that are symlinked into the working directory of the @@ -437,7 +442,8 @@ go_test = prelude_rule( "platform": attrs.option(attrs.string(), default = None), "runner": attrs.option(attrs.dep(), default = None), "specs": attrs.option(attrs.arg(json = True), default = None), - } + } | + re_test_common.test_args() ), ) diff --git a/prelude/decls/groovy_rules.bzl b/prelude/decls/groovy_rules.bzl index 0e5aef98b3302..91f71860cc229 100644 --- a/prelude/decls/groovy_rules.bzl +++ b/prelude/decls/groovy_rules.bzl @@ -131,6 +131,7 @@ groovy_library = prelude_rule( "runtime_deps": attrs.list(attrs.dep(), default = []), "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -188,6 +189,7 @@ groovy_test = prelude_rule( "use_cxx_libraries": attrs.option(attrs.bool(), default = None), "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), "vm_args": attrs.list(attrs.arg(), default = []), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) diff --git a/prelude/decls/ios_rules.bzl b/prelude/decls/ios_rules.bzl index a42dbfd9bf4e6..4a4608d7a6589 100644 --- a/prelude/decls/ios_rules.bzl +++ b/prelude/decls/ios_rules.bzl @@ -225,7 +225,8 @@ apple_binary = prelude_rule( "uses_modules": attrs.bool(default = False), "xcode_private_headers_symlinks": attrs.option(attrs.bool(), default = None), "xcode_public_headers_symlinks": attrs.option(attrs.bool(), default = None), - } + } | + buck.allow_cache_upload_arg() ), ) @@ -342,10 +343,12 @@ apple_bundle = prelude_rule( } | apple_common.info_plist_arg() | apple_common.info_plist_substitutions_arg() | + apple_common.privacy_manifest_arg() | { "asset_catalogs_compilation_options": attrs.dict(key = attrs.string(), value = attrs.any(), default = {}, doc = """ A dict holding parameters for asset catalogs compiler (actool). Its options include: - * `notices` (defaults to `True`) + + * `notices` (defaults to `True`) * `warnings` (defaults to `True`) * `errors` (defaults to `True`) * `compress_pngs` (defaults to `True`) @@ -513,7 +516,8 @@ apple_library = prelude_rule( "uses_modules": attrs.bool(default = False), "xcode_private_headers_symlinks": attrs.option(attrs.bool(), default = None), "xcode_public_headers_symlinks": attrs.option(attrs.bool(), default = None), - } + } | + buck.allow_cache_upload_arg() ), ) @@ -781,7 +785,8 @@ apple_test = prelude_rule( "xcode_private_headers_symlinks": attrs.option(attrs.bool(), default = None), "xcode_product_type": attrs.option(attrs.string(), default = None), "xcode_public_headers_symlinks": attrs.option(attrs.bool(), default = None), - } + } | + buck.allow_cache_upload_arg() ), ) diff --git a/prelude/decls/java_rules.bzl b/prelude/decls/java_rules.bzl index abc89185e5c44..2f8a0424c737a 100644 --- a/prelude/decls/java_rules.bzl +++ b/prelude/decls/java_rules.bzl @@ -290,6 +290,7 @@ java_library = prelude_rule( "proguard_config": attrs.option(attrs.source(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -423,6 +424,7 @@ java_test = prelude_rule( "test_case_timeout_ms": attrs.option(attrs.int(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -467,6 +469,7 @@ java_test_runner = prelude_rule( "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "srcs": attrs.list(attrs.source(), default = []), "target": attrs.option(attrs.string(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) diff --git a/prelude/decls/kotlin_rules.bzl b/prelude/decls/kotlin_rules.bzl index b8dce5a80f80b..b1550992a0fce 100644 --- a/prelude/decls/kotlin_rules.bzl +++ b/prelude/decls/kotlin_rules.bzl @@ -195,6 +195,7 @@ kotlin_library = prelude_rule( "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "target": attrs.option(attrs.string(), default = None), "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -300,6 +301,7 @@ kotlin_test = prelude_rule( "use_cxx_libraries": attrs.option(attrs.bool(), default = None), "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) diff --git a/prelude/decls/python_rules.bzl b/prelude/decls/python_rules.bzl index e84a135a792f9..424f4f8c22d72 100644 --- a/prelude/decls/python_rules.bzl +++ b/prelude/decls/python_rules.bzl @@ -12,6 +12,16 @@ load(":python_common.bzl", "python_common") NativeLinkStrategy = ["separate", "merged"] +def _typing_arg(): + return { + "py_version_for_type_checking": attrs.option(attrs.string(), default = None, doc = """ + This option will force the type checker to perform checking under a specific version of Python interpreter. +"""), + "typing": attrs.bool(default = True, doc = """ + Determines whether to perform type checking on the given target. Default is True. +"""), + } + cxx_python_extension = prelude_rule( name = "cxx_python_extension", docs = """ @@ -270,7 +280,8 @@ python_binary = prelude_rule( "version_universe": attrs.option(attrs.string(), default = None), "zip_safe": attrs.option(attrs.bool(), default = None), } | - buck.allow_cache_upload_arg() + buck.allow_cache_upload_arg() | + _typing_arg() ), ) @@ -339,7 +350,8 @@ python_library = prelude_rule( "versioned_resources": attrs.option(attrs.versioned(attrs.named_set(attrs.source(), sorted = True)), default = None), "versioned_srcs": attrs.option(attrs.versioned(attrs.named_set(attrs.source(), sorted = True)), default = None), "zip_safe": attrs.option(attrs.bool(), default = None), - } + } | + _typing_arg() ), ) @@ -449,7 +461,8 @@ python_test = prelude_rule( "versioned_resources": attrs.option(attrs.versioned(attrs.named_set(attrs.source(), sorted = True)), default = None), "versioned_srcs": attrs.option(attrs.versioned(attrs.named_set(attrs.source(), sorted = True)), default = None), "zip_safe": attrs.option(attrs.bool(), default = None), - } + } | + _typing_arg() ), ) diff --git a/prelude/decls/re_test_common.bzl b/prelude/decls/re_test_common.bzl index 0bc0970ffaf36..c4c4cd9343c74 100644 --- a/prelude/decls/re_test_common.bzl +++ b/prelude/decls/re_test_common.bzl @@ -17,6 +17,7 @@ def _opts_for_tests_arg() -> Attr: # "listing_capabilities": Dict | None # "use_case": str | None # "remote_cache_enabled": bool | None + # "dependencies": list> | [] # } return attrs.dict( key = attrs.string(), @@ -29,6 +30,7 @@ def _opts_for_tests_arg() -> Attr: ), attrs.string(), attrs.bool(), + attrs.list(attrs.dict(key = attrs.string(), value = attrs.string()), default = []), ), # TODO(cjhopman): I think this default does nothing, it should be deleted default = None, @@ -38,7 +40,8 @@ def _opts_for_tests_arg() -> Attr: def _action_key_provider_arg() -> Attr: if is_full_meta_repo(): - return attrs.dep(providers = [BuildModeInfo], default = "fbcode//buck2/platform/build_mode:build_mode") + default_build_mode = read_root_config("fb", "remote_execution_test_build_mode", "fbcode//buck2/platform/build_mode:build_mode") + return attrs.dep(providers = [BuildModeInfo], default = default_build_mode) else: return attrs.option(attrs.dep(providers = [BuildModeInfo]), default = None) diff --git a/prelude/decls/rust_rules.bzl b/prelude/decls/rust_rules.bzl index 331cf99484323..703436c81d8f7 100644 --- a/prelude/decls/rust_rules.bzl +++ b/prelude/decls/rust_rules.bzl @@ -9,7 +9,7 @@ load("@prelude//cxx/user:link_group_map.bzl", "link_group_map_attr") load("@prelude//rust:link_info.bzl", "RustProcMacroPlugin") load("@prelude//rust:rust_binary.bzl", "rust_binary_impl", "rust_test_impl") load("@prelude//rust:rust_library.bzl", "prebuilt_rust_library_impl", "rust_library_impl") -load(":common.bzl", "LinkableDepType", "Linkage", "buck", "prelude_rule") +load(":common.bzl", "Linkage", "buck", "prelude_rule") load(":native_common.bzl", "native_common") load(":re_test_common.bzl", "re_test_common") load(":rust_common.bzl", "rust_common", "rust_target_dep") @@ -51,6 +51,7 @@ prebuilt_rust_library = prelude_rule( 'libfoo-abc123def456.rlib' if it has symbol versioning metadata. """), } | + native_common.preferred_linkage(preferred_linkage_type = attrs.enum(Linkage, default = "any")) | rust_common.crate(crate_type = attrs.string(default = "")) | rust_common.deps_arg(is_binary = False) | { @@ -58,7 +59,6 @@ prebuilt_rust_library = prelude_rule( "default_host_platform": attrs.option(attrs.configuration_label(), default = None), "labels": attrs.list(attrs.string(), default = []), "licenses": attrs.list(attrs.source(), default = []), - "link_style": attrs.option(attrs.enum(LinkableDepType), default = None), "proc_macro": attrs.bool(default = False), } | rust_common.cxx_toolchain_arg() | @@ -100,7 +100,8 @@ _RUST_EXECUTABLE_ATTRIBUTES = { "auto_link_groups": attrs.bool(default = True), # TODO: enable distributed thinlto "enable_distributed_thinlto": attrs.bool(default = False), - "link_group": attrs.option(attrs.string(), default = None), + # Required by the rules but not supported, since Rust is auto-link groups only + "link_group": attrs.default_only(attrs.option(attrs.string(), default = None)), "link_group_map": link_group_map_attr(), "link_group_min_binary_node_count": attrs.option(attrs.int(), default = None), "rpath": attrs.bool(default = False, doc = """ diff --git a/prelude/decls/scala_rules.bzl b/prelude/decls/scala_rules.bzl index 093ac8e5a2c02..80c95edda63ad 100644 --- a/prelude/decls/scala_rules.bzl +++ b/prelude/decls/scala_rules.bzl @@ -46,6 +46,7 @@ scala_library = prelude_rule( "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "srcs": attrs.list(attrs.source(), default = []), "target": attrs.option(attrs.string(), default = None), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) @@ -103,6 +104,7 @@ scala_test = prelude_rule( "use_cxx_libraries": attrs.option(attrs.bool(), default = None), "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), "vm_args": attrs.list(attrs.arg(), default = []), + "_wip_java_plugin_arguments": attrs.dict(attrs.label(), attrs.list(attrs.string()), default = {}), } ), ) diff --git a/prelude/erlang/common_test/common/include/buck_ct_records.hrl b/prelude/erlang/common_test/common/include/buck_ct_records.hrl index 071d643543da4..1f1ebdd3ef034 100644 --- a/prelude/erlang/common_test/common/include/buck_ct_records.hrl +++ b/prelude/erlang/common_test/common/include/buck_ct_records.hrl @@ -14,6 +14,7 @@ providers :: [{atom(), [term()]}], ct_opts :: [term()], erl_cmd :: string(), + extra_flags :: [string()], common_app_env :: #{string() => string()}, artifact_annotation_mfa :: artifact_annotations:annotation_function() }). @@ -31,6 +32,7 @@ ct_opts :: [term()], common_app_env :: #{string() => string()}, erl_cmd :: string(), + extra_flags :: [string()], artifact_annotation_mfa :: artifact_annotations:annotation_function() }). diff --git a/prelude/erlang/common_test/test_binary/src/test_binary.erl b/prelude/erlang/common_test/test_binary/src/test_binary.erl index ea35b51332c1c..2403db9f2e700 100644 --- a/prelude/erlang/common_test/test_binary/src/test_binary.erl +++ b/prelude/erlang/common_test/test_binary/src/test_binary.erl @@ -31,16 +31,7 @@ main([TestInfoFile, "list", OutputDir]) -> after test_logger:flush() end, - init:stop(ExitCode), - receive - after ?INIT_STOP_TIMEOUT -> - ?LOG_ERROR( - io_lib:format("~p failed to terminate within ~c millisecond", [ - ?MODULE, ?INIT_STOP_TIMEOUT - ]) - ), - erlang:halt(ExitCode) - end; + erlang:halt(ExitCode); main([TestInfoFile, "run", OutputDir | Tests]) -> test_logger:set_up_logger(OutputDir, test_runner), ExitCode = @@ -55,16 +46,7 @@ main([TestInfoFile, "run", OutputDir | Tests]) -> after test_logger:flush() end, - init:stop(ExitCode), - receive - after ?INIT_STOP_TIMEOUT -> - ?LOG_ERROR( - io_lib:format("~p failed to terminate within ~c millisecond", [ - ?MODULE, ?INIT_STOP_TIMEOUT - ]) - ), - erlang:halt(ExitCode) - end; + erlang:halt(ExitCode); main([TestInfoFile]) -> %% without test runner support we run all tests and need to create our own test dir OutputDir = string:trim(os:cmd("mktemp -d")), @@ -105,6 +87,7 @@ load_test_info(TestInfoFile) -> "ct_opts" := CtOpts, "extra_ct_hooks" := ExtraCtHooks, "erl_cmd" := ErlCmd, + "extra_flags" := ExtraFlags, "artifact_annotation_mfa" := ArtifactAnnotationMFA, "common_app_env" := CommonAppEnv } @@ -122,6 +105,7 @@ load_test_info(TestInfoFile) -> artifact_annotation_mfa = parse_mfa(ArtifactAnnotationMFA), ct_opts = CtOpts1, erl_cmd = ErlCmd, + extra_flags = ExtraFlags, common_app_env = CommonAppEnv }. diff --git a/prelude/erlang/common_test/test_binary/src/test_runner.erl b/prelude/erlang/common_test/test_binary/src/test_runner.erl index a85c13f707f37..a022ec418c92d 100644 --- a/prelude/erlang/common_test/test_binary/src/test_runner.erl +++ b/prelude/erlang/common_test/test_binary/src/test_runner.erl @@ -41,6 +41,7 @@ run_tests(Tests, #test_info{} = TestInfo, OutputDir, Listing) -> ct_opts = TestInfo#test_info.ct_opts, common_app_env = TestInfo#test_info.common_app_env, erl_cmd = TestInfo#test_info.erl_cmd, + extra_flags = TestInfo#test_info.extra_flags, artifact_annotation_mfa = TestInfo#test_info.artifact_annotation_mfa }) end. diff --git a/prelude/erlang/common_test/test_cli_lib/src/test.erl b/prelude/erlang/common_test/test_cli_lib/src/test.erl index 88786b1d1104b..52420ece21835 100644 --- a/prelude/erlang/common_test/test_cli_lib/src/test.erl +++ b/prelude/erlang/common_test/test_cli_lib/src/test.erl @@ -34,7 +34,9 @@ start_shell/0 ]). --type run_spec() :: string() | non_neg_integer() | [#{name := string(), suite := string()}]. +-type test_id() :: string() | non_neg_integer(). +-type test_info() :: #{name := string(), suite := atom()}. +-type run_spec() :: test_id() | [test_info()]. -type run_result() :: {non_neg_integer(), non_neg_integer()}. -spec start() -> ok. @@ -75,7 +77,7 @@ help() -> io:format("For more information, use the built in help, e.g. h(test, help)~n"), ok. --spec print_help(function(), arity()) -> ok. +-spec print_help(Fun :: atom(), arity()) -> ok. print_help(Fun, Arity) -> #{args := Args, desc := [DescFirst | DescRest]} = command_description(Fun, Arity), FunSig = string:pad( @@ -83,9 +85,10 @@ print_help(Fun, Arity) -> ), io:format("~s -- ~s~n", [FunSig, DescFirst]), Padding = string:pad("", 34), - [io:format("~s~s~n", [Padding, DescLine]) || DescLine <- DescRest]. + [io:format("~s~s~n", [Padding, DescLine]) || DescLine <- DescRest], + ok. --spec command_description(module(), arity()) -> #{args := [string()], desc := string()}. +-spec command_description(Fun :: atom(), arity()) -> #{args := [string()], desc := [string()]}. command_description(help, 0) -> #{args => [], desc => ["print help"]}; command_description(info, 0) -> @@ -130,7 +133,7 @@ command_description(F, A) -> %% @doc List all available tests %% @equiv test:list("") --spec list() -> non_neg_integer(). +-spec list() -> ok | {error, term()}. list() -> list(""). @@ -138,26 +141,27 @@ list() -> %% [https://www.erlang.org/doc/man/re.html#regexp_syntax] for the supported %% regular expression syntax. If a module is given as argument, list all %% tests from that module instead --spec list(RegExOrModule :: module() | string()) -> non_neg_integer(). +-spec list(RegExOrModule :: module() | string()) -> ok | {error, term()}. list(RegEx) when is_list(RegEx) -> ensure_initialized(), - Tests = ct_daemon:list(RegEx), - print_tests(Tests). + case ct_daemon:list(RegEx) of + {invalid_regex, _} = Err -> {error, Err}; + Tests -> print_tests(Tests) + end. %% @doc Run a test given by either the test id from the last list() command, or %% a regex that matches exactly one test. Tests are run with the shortest possible %% setup. This call does not recompile the test suite and its dependencies, but %% runs them as is. You can manually recompile code with c(Module). %% To reset the test state use reset(). --spec rerun(string() | non_neg_integer() | [#{name := string(), suite := string()}]) -> - run_result(). +-spec rerun(run_spec()) -> run_result(). rerun(Spec) -> ensure_initialized(), do_plain_test_run(Spec). %% @doc update code and run all tests %% @equiv run("") --spec run() -> ok | error. +-spec run() -> run_result() | error. run() -> run(""). @@ -177,8 +181,15 @@ run(RegExOrId) -> ok -> io:format("Reloading all changed modules... "), Loaded = ct_daemon:load_changed(), - io:format("reloaded ~p modules ~P~n", [erlang:length(Loaded), Loaded, 10]), - rerun(ToRun); + case erlang:length(Loaded) of + 0 -> + do_plain_test_run(ToRun); + ChangedCount -> + io:format("reloaded ~p modules ~P~n", [ChangedCount, Loaded, 10]), + % There were some changes, so list the tests again, then run but without recompiling changes + % Note that if called with the RegEx insted of ToRun test list like above, do_plain_test_run/1 will list the tests again + do_plain_test_run(RegExOrId) + end; Error -> Error end @@ -216,6 +227,7 @@ ensure_initialized() -> ok end. +-spec init_utility_apps() -> boolean(). init_utility_apps() -> RunningApps = proplists:get_value(running, application:info()), case proplists:is_defined(test_cli_lib, RunningApps) of @@ -233,6 +245,7 @@ init_utility_apps() -> end end. +-spec init_node() -> boolean(). init_node() -> case ct_daemon:alive() of true -> @@ -259,6 +272,7 @@ init_node() -> true end. +-spec watchdog() -> no_return(). watchdog() -> Node = ct_daemon_node:get_node(), true = erlang:monitor_node(Node, true), @@ -272,6 +286,7 @@ watchdog() -> erlang:halt() end. +-spec init_group_leader() -> boolean(). init_group_leader() -> %% set the group leader unconditionally, we need to do this since %% during init, the group leader is different then the one from the @@ -279,11 +294,13 @@ init_group_leader() -> ct_daemon:set_gl(), false. +-spec print_tests([{module(), [{non_neg_integer(), string()}]}]) -> ok. print_tests([]) -> io:format("no tests found~n"); print_tests(Tests) -> print_tests_impl(lists:reverse(Tests)). +-spec print_tests_impl([{module(), [{non_neg_integer(), string()}]}]) -> ok. print_tests_impl([]) -> ok; print_tests_impl([{Suite, SuiteTests} | Rest]) -> @@ -293,9 +310,12 @@ print_tests_impl([{Suite, SuiteTests} | Rest]) -> -spec is_debug_session() -> boolean(). is_debug_session() -> - application:get_env(test_cli_lib, debugger_mode, false). + case application:get_env(test_cli_lib, debugger_mode, false) of + Value when is_boolean(Value) -> + Value + end. --spec collect_results(#{module => [string()]}) -> #{string => ct_daemon_core:run_result()}. +-spec collect_results(#{module => [string()]}) -> #{string() => ct_daemon_core:run_result()}. collect_results(PerSuite) -> maps:fold( fun(Suite, Tests, Acc) -> @@ -330,7 +350,7 @@ ensure_per_suite_encapsulation(Suite) -> end end. --spec discover(string() | non_neg_integer()) -> [#{name := string(), suite := string()}]. +-spec discover(string() | non_neg_integer()) -> [test_info()]. discover(RegExOrId) -> case ct_daemon:discover(RegExOrId) of {error, not_listed_yet} -> @@ -375,11 +395,12 @@ do_plain_test_run(RegExOrId) -> ToRun -> do_plain_test_run(ToRun) end. --spec start_shell() -> no_return(). +-spec start_shell() -> ok | {error, term()}. start_shell() -> case string:to_integer(erlang:system_info(otp_release)) of {Version, _} when Version >= 26 -> shell:start_interactive(); _ -> - user_drv:start() + user_drv:start(), + ok end. diff --git a/prelude/erlang/common_test/test_exec/BUCK.v2 b/prelude/erlang/common_test/test_exec/BUCK.v2 index 0ea358626cf37..af2feb4f4da4e 100644 --- a/prelude/erlang/common_test/test_exec/BUCK.v2 +++ b/prelude/erlang/common_test/test_exec/BUCK.v2 @@ -23,3 +23,15 @@ erlang_application( use_global_parse_transforms = False, visibility = ["PUBLIC"], ) + +erlang_tests( + contacts = ["whatsapp_erlclient"], + labels = ["unit"], + suites = glob( + ["test/*_SUITE.erl"], + ), + deps = [ + "stdlib", + ":test_exec", + ], +) diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon.erl index b5a4fb7fe7166..c2a1aa6d18e7f 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon.erl @@ -89,7 +89,7 @@ list(RegEx) -> end. -spec discover(pos_integer() | string()) -> - #{suite := module(), name := string()} + [#{suite := module(), name := string()}] | ct_daemon_runner:discover_error(). discover(RegExOrId) -> do_call({discover, RegExOrId}). diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl index d260bc249210e..bbe7810c5e68b 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl @@ -73,6 +73,7 @@ start( % see T129435667 Port = ct_runner:start_test_node( os:find_executable("erl"), + [], CodePaths, ConfigFiles, OutputDir, diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_printer.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_printer.erl index dd81f8cbde9f8..dc8f945d9c609 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_printer.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_printer.erl @@ -55,6 +55,10 @@ print_result(Name, {error, {_TestId, {'ct_daemon_core$sentinel_crash', Info}}}) io:format("~ts ~ts~n", [?CROSS_MARK, Name]), io:format("Test process received EXIT signal with reason: ~p~n", [Info]), fail; +print_result(Name, {error, {_TestId, {timetrap, TimeoutValue}}}) -> + io:format("~ts ~ts~n", [?CROSS_MARK, Name]), + io:format("Test timed out after ~p ms~n", [TimeoutValue]), + fail; print_result(Name, Unstructured) -> io:format("~ts ~ts~n", [?CROSS_MARK, Name]), io:format("unable to format failure reason, please report.~n"), diff --git a/prelude/erlang/common_test/test_exec/src/ct_executor.erl b/prelude/erlang/common_test/test_exec/src/ct_executor.erl index fa61612be61f1..af35d10dfa4c9 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_executor.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_executor.erl @@ -11,15 +11,22 @@ %% Notably allows us to call post/pre method on the node if needed, e.g for coverage. -module(ct_executor). - -include_lib("kernel/include/logger.hrl"). -include_lib("common/include/buck_ct_records.hrl"). +-compile(warn_missing_spec_all). -export([run/1]). -% Time we give the beam to close off, in ms. --define(INIT_STOP_TIMEOUT, 5000). +%% `ct_run_arg()` represents an option accepted by ct:run_test/1, such as +%% `multiply_timetraps` or `ct_hooks`. +%% For all the options, see https://www.erlang.org/doc/man/ct#run_test-1 +-type ct_run_arg() :: {atom(), term()}. +-type ct_exec_arg() :: {output_dir | suite | providers, term()}. + +% For testing +-export([split_args/1]). +-spec run([string()]) -> no_return(). run(Args) when is_list(Args) -> ExitCode = try @@ -88,23 +95,9 @@ run(Args) when is_list(Args) -> io:format("~ts\n", [erl_error:format_exception(Class1, Reason1, Stack1)]), 1 end, - case ExitCode of - 0 -> - init:stop(0), - receive - after ?INIT_STOP_TIMEOUT -> - ?LOG_ERROR( - io_lib:format("~p failed to terminate within ~c millisecond", [ - ?MODULE, ?INIT_STOP_TIMEOUT - ]) - ), - erlang:halt(0) - end; - _ -> - erlang:halt(ExitCode) - end. + erlang:halt(ExitCode). --spec parse_arguments([string()]) -> {proplists:proplist(), [term()]}. +-spec parse_arguments([string()]) -> {[ct_exec_arg()], [ct_run_arg()]}. parse_arguments(Args) -> % The logger is not set up yet. % This will be sent to the program executing it (ct_runner), @@ -123,14 +116,27 @@ parse_arguments(Args) -> split_args(ParsedArgs). % @doc Splits the argument before those that happens -% before ct_args (the executor args) amd those after -% (the args for ct_run). -split_args(Args) -> split_args(Args, [], []). +% before ct_args (the executor args) and those after +% (the args for ct_run). ct_args will always be +% present in the list +-spec split_args([term()]) -> {[ct_exec_arg()], [ct_run_arg()]}. +split_args(Args) -> + {CtExecutorArgs, [ct_args | CtRunArgs]} = lists:splitwith(fun(Arg) -> Arg =/= ct_args end, Args), + {parse_ct_exec_args(CtExecutorArgs), parse_ct_run_args(CtRunArgs)}. + +-spec parse_ct_run_args([term()]) -> [ct_run_arg()]. +parse_ct_run_args([]) -> + []; +parse_ct_run_args([{Key, _Value} = Arg | Args]) when is_atom(Key) -> + [Arg | parse_ct_run_args(Args)]. -split_args([ct_args | Args], CtExecutorArgs, []) -> {lists:reverse(CtExecutorArgs), Args}; -split_args([Arg | Args], CtExecutorArgs, []) -> split_args(Args, [Arg | CtExecutorArgs], []); -split_args([], CtExecutorArgs, []) -> {lists:reverse(CtExecutorArgs), []}. +-spec parse_ct_exec_args([term()]) -> [ct_exec_arg()]. +parse_ct_exec_args([]) -> + []; +parse_ct_exec_args([{Key, _Value} = Arg | Args]) when Key =:= output_dir; Key =:= suite; Key =:= providers -> + [Arg | parse_ct_exec_args(Args)]. +-spec debug_print(string(), [term()]) -> ok. debug_print(Fmt, Args) -> case os:getenv("ERLANG_BUCK_DEBUG_PRINT") of false -> io:format(Fmt, Args); diff --git a/prelude/erlang/common_test/test_exec/src/ct_runner.erl b/prelude/erlang/common_test/test_exec/src/ct_runner.erl index 1b5dacf406da7..5240f28b1c6fd 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_runner.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_runner.erl @@ -28,8 +28,8 @@ ]). -export([ - start_test_node/5, start_test_node/6, + start_test_node/7, cookie/0, generate_arg_tuple/2, project_root/0 @@ -144,6 +144,7 @@ run_test( providers = Providers, suite = Suite, erl_cmd = ErlCmd, + extra_flags = ExtraFlags, common_app_env = CommonAppEnv } = _TestEnv, PortEpmd @@ -159,6 +160,7 @@ run_test( start_test_node( ErlCmd, + ExtraFlags, CodePath, ConfigFiles, OutputDir, @@ -209,25 +211,50 @@ common_app_env_args(Env) -> -spec start_test_node( Erl :: string(), + ExtraFlags :: [string()], CodePath :: [file:filename_all()], ConfigFiles :: [file:filename_all()], OutputDir :: file:filename_all(), PortSettings :: port_settings() ) -> port(). -start_test_node(ErlCmd, CodePath, ConfigFiles, OutputDir, PortSettings0) -> - start_test_node(ErlCmd, CodePath, ConfigFiles, OutputDir, PortSettings0, false). +start_test_node( + ErlCmd, + ExtraFlags, + CodePath, + ConfigFiles, + OutputDir, + PortSettings0 +) -> + start_test_node( + ErlCmd, + ExtraFlags, + CodePath, + ConfigFiles, + OutputDir, + PortSettings0, + false + ). -spec start_test_node( Erl :: string(), + ExtraFlags :: [string()], CodePath :: [file:filename_all()], ConfigFiles :: [file:filename_all()], OutputDir :: file:filename_all(), PortSettings :: port_settings(), ReplayIo :: boolean() ) -> port(). -start_test_node(ErlCmd, CodePath, ConfigFiles, OutputDir, PortSettings0, ReplayIo) -> +start_test_node( + ErlCmd, + ExtraFlags, + CodePath, + ConfigFiles, + OutputDir, + PortSettings0, + ReplayIo +) -> % split of args from Erl which can contain emulator flags - [_Executable | ExtraFlags] = string:split(ErlCmd, " ", all), + [_Executable | Flags] = string:split(ErlCmd, " ", all), % we ignore the executable we got, and use the erl command from the % toolchain that executes this code ErlExecutable = os:find_executable("erl"), @@ -237,7 +264,7 @@ start_test_node(ErlCmd, CodePath, ConfigFiles, OutputDir, PortSettings0, ReplayI %% merge args, enc, cd settings LaunchArgs = - ExtraFlags ++ + Flags ++ ExtraFlags ++ build_common_args(CodePath, ConfigFiles) ++ proplists:get_value(args, PortSettings0, []), diff --git a/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl b/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl new file mode 100644 index 0000000000000..bcf4d0b86600e --- /dev/null +++ b/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl @@ -0,0 +1,41 @@ +%% Copyright (c) Meta Platforms, Inc. and affiliates. +%% This source code is licensed under both the MIT license found in the +%% LICENSE-MIT file in the root directory of this source tree and the Apache +%% License, Version 2.0 found in the LICENSE-APACHE file in the root directory +%% of this source tree. +%%% % @format +-module(ct_executor_SUITE). + +-include_lib("stdlib/include/assert.hrl"). + +-export([all/0]). + +-export([ + test_split_args/1 +]). + +all() -> + [test_split_args]. + +test_split_args(_Config) -> + ?assertEqual( + {[{output_dir, ""}, {providers, [something]}, {suite, a_suite}], [{dir, ""}, {suite, a_suite}, {group, a_group}]}, + ct_executor:split_args([ + {output_dir, ""}, + {providers, [something]}, + {suite, a_suite}, + ct_args, + {dir, ""}, + {suite, a_suite}, + {group, a_group} + ]) + ), + ?assertEqual( + {[{output_dir, ""}, {providers, [something]}, {suite, a_suite}], []}, + ct_executor:split_args([{output_dir, ""}, {providers, [something]}, {suite, a_suite}, ct_args]) + ), + ?assertEqual( + {[], [{dir, ""}, {suite, a_suite}, {group, a_group}]}, + ct_executor:split_args([ct_args, {dir, ""}, {suite, a_suite}, {group, a_group}]) + ), + ?assertEqual({[], []}, ct_executor:split_args([ct_args])). diff --git a/prelude/erlang/erlang_application.bzl b/prelude/erlang/erlang_application.bzl index e9d3a134bf837..22f9daf9877b9 100644 --- a/prelude/erlang/erlang_application.bzl +++ b/prelude/erlang/erlang_application.bzl @@ -37,7 +37,6 @@ load( "multidict_projection", "multidict_projection_key", "normalise_metadata", - "str_to_bool", "to_term_args", ) @@ -366,7 +365,7 @@ def link_output( def _link_srcs_folder(ctx: AnalysisContext) -> dict[str, Artifact]: """Build mapping for the src folder if erlang.include_src is set""" - if not str_to_bool(read_root_config("erlang", "include_src", "False")): + if not ctx.attrs.include_src: return {} srcs = { paths.join("src", src_file.basename): src_file diff --git a/prelude/erlang/erlang_tests.bzl b/prelude/erlang/erlang_tests.bzl index 7a7d8a3647ca0..2fda9199372e4 100644 --- a/prelude/erlang/erlang_tests.bzl +++ b/prelude/erlang/erlang_tests.bzl @@ -272,6 +272,7 @@ def _write_test_info_file( "dependencies": _list_code_paths(dependencies), "erl_cmd": cmd_args(['"', cmd_args(erl_cmd, delimiter = " "), '"'], delimiter = ""), "extra_ct_hooks": ctx.attrs.extra_ct_hooks, + "extra_flags": ctx.attrs.extra_erl_flags, "providers": ctx.attrs._providers, "test_dir": test_dir, "test_suite": test_suite, diff --git a/prelude/genrule.bzl b/prelude/genrule.bzl index a4e1e63365434..a7963df8ff3fb 100644 --- a/prelude/genrule.bzl +++ b/prelude/genrule.bzl @@ -10,11 +10,9 @@ load("@prelude//:cache_mode.bzl", "CacheModeInfo") load("@prelude//:genrule_local_labels.bzl", "genrule_labels_require_local") load("@prelude//:genrule_toolchain.bzl", "GenruleToolchainInfo") -load("@prelude//:genrule_types.bzl", "GENRULE_MARKER_SUBTARGET_NAME", "GenruleMarkerInfo") load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") load("@prelude//android:build_only_native_code.bzl", "is_build_only_native_code") load("@prelude//os_lookup:defs.bzl", "OsLookup") -load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:utils.bzl", "flatten", "value_or") GENRULE_OUT_DIR = "out" @@ -347,14 +345,7 @@ def process_genrule( **metadata_args ) - # Use a subtarget to insert a marker, as callsites make assumptions about - # the providers of `process_genrule()`. We want to have the marker in - # `DefaultInfo` rather than in `genrule_impl()` because we want to identify - # all classes of genrule-like rules. sub_targets = {k: [DefaultInfo(default_outputs = v)] for (k, v) in named_outputs.items()} - expect(GENRULE_MARKER_SUBTARGET_NAME not in sub_targets, "Conflicting private `{}` subtarget and named output".format(GENRULE_MARKER_SUBTARGET_NAME)) - sub_targets[GENRULE_MARKER_SUBTARGET_NAME] = [GenruleMarkerInfo()] - providers = [DefaultInfo( default_outputs = default_outputs, sub_targets = sub_targets, diff --git a/prelude/genrule_local_labels.bzl b/prelude/genrule_local_labels.bzl index 13cb97fd20d27..428f61d766230 100644 --- a/prelude/genrule_local_labels.bzl +++ b/prelude/genrule_local_labels.bzl @@ -170,10 +170,12 @@ _GENRULE_LOCAL_LABELS = {label: True for label in [ # Some Qt genrules don't support RE yet "qt_moc", - "qt_qrc_gen", + "qt_qmlcachegen", "qt_qrc_compile", + "qt_qrc_gen", "qt_qsb_gen", - "qt_qmlcachegen", + "qt_rcc", + "qt_uic", # use local jar "uses_jar", diff --git a/prelude/go/cgo_library.bzl b/prelude/go/cgo_library.bzl index 56aeed9a69dde..217d203fed2ae 100644 --- a/prelude/go/cgo_library.bzl +++ b/prelude/go/cgo_library.bzl @@ -49,6 +49,7 @@ load( "map_idx", ) load(":compile.bzl", "GoPkgCompileInfo", "compile", "get_filtered_srcs", "get_inherited_compile_pkgs") +load(":coverage.bzl", "GoCoverageMode", "cover_srcs") load(":link.bzl", "GoPkgLinkInfo", "get_inherited_link_pkgs") load(":packages.bzl", "GoPkg", "go_attr_pkg_name", "merge_pkgs") load(":toolchain.bzl", "GoToolchainInfo", "get_toolchain_cmd_args") @@ -101,12 +102,14 @@ def _cgo( args.add(cmd_args(go_toolchain.cgo, format = "--cgo={}")) c_compiler = cxx_toolchain.c_compiler_info - linker = cxx_toolchain.linker_info + # linker = cxx_toolchain.linker_info - ldflags = cmd_args( - linker.linker_flags, - go_toolchain.external_linker_flags, - ) + # Passing fbcode-platform ldflags may create S365277, so I would + # comment this change until we really need to do it. + # ldflags = cmd_args( + # linker.linker_flags, + # go_toolchain.external_linker_flags, + # ) # Construct the full C/C++ command needed to preprocess/compile sources. cxx_cmd = cmd_args() @@ -115,10 +118,7 @@ def _cgo( cxx_cmd.add(c_compiler.compiler_flags) cxx_cmd.add(pre_args) cxx_cmd.add(pre_include_dirs) - - # Passing the same value as go-build, because our -g flags break cgo - # in some buck modes - cxx_cmd.add("-g") + cxx_cmd.add(go_toolchain.c_compiler_flags) # Wrap the C/C++ command in a wrapper script to avoid arg length limits. is_win = ctx.attrs._exec_os_type[OsLookup].platform == "windows" @@ -136,7 +136,6 @@ def _cgo( is_executable = True, ) args.add(cmd_args(cxx_wrapper, format = "--env-cc={}")) - args.add(cmd_args(ldflags, format = "--env-ldflags={}")) args.hidden(cxx_cmd) # TODO(agallagher): cgo outputs a dir with generated sources, but I'm not @@ -158,6 +157,20 @@ def _cgo( return go_srcs, c_headers, c_srcs +def _compile_with_coverage(ctx: AnalysisContext, pkg_name: str, srcs: cmd_args, coverage_mode: GoCoverageMode, shared: bool = False) -> (Artifact, cmd_args): + cov_res = cover_srcs(ctx, pkg_name, coverage_mode, srcs, shared) + srcs = cov_res.srcs + coverage_vars = cov_res.variables + coverage_pkg = compile( + ctx, + pkg_name, + srcs = srcs, + deps = ctx.attrs.deps + ctx.attrs.exported_deps, + coverage_mode = coverage_mode, + shared = shared, + ) + return (coverage_pkg, coverage_vars) + def cgo_library_impl(ctx: AnalysisContext) -> list[Provider]: pkg_name = go_attr_pkg_name(ctx) @@ -225,25 +238,19 @@ def cgo_library_impl(ctx: AnalysisContext) -> list[Provider]: all_srcs.add(get_filtered_srcs(ctx, ctx.attrs.go_srcs)) # Build Go library. - static_pkg = compile( - ctx, - pkg_name, - all_srcs, - deps = ctx.attrs.deps + ctx.attrs.exported_deps, - shared = False, - ) - shared_pkg = compile( + compiled_pkg = compile( ctx, pkg_name, all_srcs, deps = ctx.attrs.deps + ctx.attrs.exported_deps, - shared = True, + shared = ctx.attrs._compile_shared, ) + pkg_with_coverage = {mode: _compile_with_coverage(ctx, pkg_name, all_srcs, mode) for mode in GoCoverageMode} pkgs = { pkg_name: GoPkg( - shared = shared_pkg, - static = static_pkg, cgo = True, + pkg = compiled_pkg, + pkg_with_coverage = pkg_with_coverage, ), } @@ -252,7 +259,7 @@ def cgo_library_impl(ctx: AnalysisContext) -> list[Provider]: # to work with cgo. And when nearly every FB service client is cgo, # we need to support it well. return [ - DefaultInfo(default_output = static_pkg, other_outputs = go_srcs), + DefaultInfo(default_output = compiled_pkg, other_outputs = go_srcs), GoPkgCompileInfo(pkgs = merge_pkgs([ pkgs, get_inherited_compile_pkgs(ctx.attrs.exported_deps), diff --git a/prelude/go/compile.bzl b/prelude/go/compile.bzl index ffb4f25cd7f27..f33f1e7614620 100644 --- a/prelude/go/compile.bzl +++ b/prelude/go/compile.bzl @@ -6,12 +6,17 @@ # of this source tree. load("@prelude//:paths.bzl", "paths") + +# @unused this comment is to make the linter happy. The linter thinks +# GoCoverageMode is unused despite it being used in the function signature of +# multiple functions. +load(":coverage.bzl", "GoCoverageMode") load( ":packages.bzl", "GoPkg", # @Unused used as type + "make_importcfg", "merge_pkgs", "pkg_artifacts", - "stdlib_pkg_artifacts", ) load(":toolchain.bzl", "GoToolchainInfo", "get_toolchain_cmd_args") @@ -31,8 +36,13 @@ GoTestInfo = provider( }, ) -def _out_root(shared: bool = False): - return "__shared__" if shared else "__static__" +def _out_root(shared: bool = False, coverage_mode: [GoCoverageMode, None] = None): + path = "static" + if shared: + path = "shared" + if coverage_mode: + path += "__coverage_" + coverage_mode.value + "__" + return path def get_inherited_compile_pkgs(deps: list[Dependency]) -> dict[str, GoPkg]: return merge_pkgs([d[GoPkgCompileInfo].pkgs for d in deps if GoPkgCompileInfo in d]) @@ -72,6 +82,7 @@ def _assemble_cmd( go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] cmd = cmd_args() cmd.add(go_toolchain.assembler) + cmd.add(go_toolchain.assembler_flags) cmd.add(flags) cmd.add("-p", pkg_name) if shared: @@ -81,15 +92,18 @@ def _assemble_cmd( def _compile_cmd( ctx: AnalysisContext, + root: str, pkg_name: str, pkgs: dict[str, Artifact] = {}, deps: list[Dependency] = [], flags: list[str] = [], - shared: bool = False) -> cmd_args: + shared: bool = False, + coverage_mode: [GoCoverageMode, None] = None) -> cmd_args: go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] cmd = cmd_args() cmd.add(go_toolchain.compiler) + cmd.add(go_toolchain.compiler_flags) cmd.add("-p", pkg_name) cmd.add("-pack") cmd.add("-nolocalimports") @@ -99,33 +113,16 @@ def _compile_cmd( # Add shared/static flags. if shared: cmd.add("-shared") - cmd.add(go_toolchain.compiler_flags_shared) - else: - cmd.add(go_toolchain.compiler_flags_static) # Add Go pkgs inherited from deps to compiler search path. all_pkgs = merge_pkgs([ pkgs, - pkg_artifacts(get_inherited_compile_pkgs(deps), shared = shared), - stdlib_pkg_artifacts(go_toolchain, shared = shared), + pkg_artifacts(get_inherited_compile_pkgs(deps), coverage_mode = coverage_mode), ]) - importcfg_content = [] - for name_, pkg_ in all_pkgs.items(): - # Hack: we use cmd_args get "artifact" valid path and write it to a file. - importcfg_content.append(cmd_args("packagefile ", name_, "=", pkg_, delimiter = "")) - - # Future work: support importmap in buck rules insted of hacking here. - if name_.startswith("third-party-source/go/"): - real_name_ = name_.removeprefix("third-party-source/go/") - importcfg_content.append(cmd_args("importmap ", real_name_, "=", name_, delimiter = "")) - - root = _out_root(shared) - importcfg = ctx.actions.declare_output(root, paths.basename(pkg_name) + "-importcfg") - ctx.actions.write(importcfg.as_output(), importcfg_content) + importcfg = make_importcfg(ctx, root, pkg_name, all_pkgs, with_importmap = True) cmd.add("-importcfg", importcfg) - cmd.hidden(all_pkgs.values()) return cmd @@ -137,15 +134,16 @@ def compile( deps: list[Dependency] = [], compile_flags: list[str] = [], assemble_flags: list[str] = [], - shared: bool = False) -> Artifact: + shared: bool = False, + coverage_mode: [GoCoverageMode, None] = None) -> Artifact: go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] - root = _out_root(shared) + root = _out_root(shared, coverage_mode) output = ctx.actions.declare_output(root, paths.basename(pkg_name) + ".a") cmd = get_toolchain_cmd_args(go_toolchain) cmd.add(go_toolchain.compile_wrapper[RunInfo]) cmd.add(cmd_args(output.as_output(), format = "--output={}")) - cmd.add(cmd_args(_compile_cmd(ctx, pkg_name, pkgs, deps, compile_flags, shared = shared), format = "--compiler={}")) + cmd.add(cmd_args(_compile_cmd(ctx, root, pkg_name, pkgs, deps, compile_flags, shared = shared, coverage_mode = coverage_mode), format = "--compiler={}")) cmd.add(cmd_args(_assemble_cmd(ctx, pkg_name, assemble_flags, shared = shared), format = "--assembler={}")) cmd.add(cmd_args(go_toolchain.packer, format = "--packer={}")) if ctx.attrs.embedcfg: @@ -160,6 +158,9 @@ def compile( identifier = paths.basename(pkg_name) if shared: identifier += "[shared]" + if coverage_mode: + identifier += "[coverage_" + coverage_mode.value + "]" + ctx.actions.run(cmd, category = "go_compile", identifier = identifier) return output diff --git a/prelude/go/constraints/BUCK.v2 b/prelude/go/constraints/BUCK.v2 new file mode 100644 index 0000000000000..a4b034fe7a14d --- /dev/null +++ b/prelude/go/constraints/BUCK.v2 @@ -0,0 +1,39 @@ +constraint_setting( + name = "cgo_enabled", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "cgo_enabled_auto", + constraint_setting = ":cgo_enabled", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "cgo_enabled_true", + constraint_setting = ":cgo_enabled", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "cgo_enabled_false", + constraint_setting = ":cgo_enabled", + visibility = ["PUBLIC"], +) + +constraint_setting( + name = "compile_shared", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "compile_shared_false", + constraint_setting = ":compile_shared", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "compile_shared_true", + constraint_setting = ":compile_shared", + visibility = ["PUBLIC"], +) diff --git a/prelude/go/coverage.bzl b/prelude/go/coverage.bzl index b85f6845386fc..102a3502a704b 100644 --- a/prelude/go/coverage.bzl +++ b/prelude/go/coverage.bzl @@ -23,10 +23,13 @@ GoCoverResult = record( variables = field(cmd_args), ) -def cover_srcs(ctx: AnalysisContext, pkg_name: str, mode: GoCoverageMode, srcs: cmd_args) -> GoCoverResult: - out_covered_src_dir = ctx.actions.declare_output("__covered_srcs__", dir = True) - out_srcs_argsfile = ctx.actions.declare_output("covered_srcs.txt") - out_coverage_vars_argsfile = ctx.actions.declare_output("coverage_vars.txt") +def cover_srcs(ctx: AnalysisContext, pkg_name: str, mode: GoCoverageMode, srcs: cmd_args, shared: bool) -> GoCoverResult: + path = "static_" + mode.value + if shared: + path = "shared_" + mode.value + out_covered_src_dir = ctx.actions.declare_output("__covered_" + path + "_srcs__", dir = True) + out_srcs_argsfile = ctx.actions.declare_output("covered_" + path + "_srcs.txt") + out_coverage_vars_argsfile = ctx.actions.declare_output("coverage_" + path + "_vars.txt") go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] cmd = cmd_args() @@ -38,7 +41,7 @@ def cover_srcs(ctx: AnalysisContext, pkg_name: str, mode: GoCoverageMode, srcs: cmd.add("--out-srcs-argsfile", out_srcs_argsfile.as_output()) cmd.add("--pkg-name", pkg_name) cmd.add(srcs) - ctx.actions.run(cmd, category = "go_cover") + ctx.actions.run(cmd, category = "go_cover_" + path) return GoCoverResult( srcs = cmd_args(out_srcs_argsfile, format = "@{}").hidden(out_covered_src_dir).hidden(srcs), diff --git a/prelude/go/go_library.bzl b/prelude/go/go_library.bzl index 2bc66e990cc27..07bc8f4634597 100644 --- a/prelude/go/go_library.bzl +++ b/prelude/go/go_library.bzl @@ -24,9 +24,25 @@ load( "map_idx", ) load(":compile.bzl", "GoPkgCompileInfo", "GoTestInfo", "compile", "get_filtered_srcs", "get_inherited_compile_pkgs") +load(":coverage.bzl", "GoCoverageMode", "cover_srcs") load(":link.bzl", "GoPkgLinkInfo", "get_inherited_link_pkgs") load(":packages.bzl", "GoPkg", "go_attr_pkg_name", "merge_pkgs") +def _compile_with_coverage(ctx: AnalysisContext, pkg_name: str, srcs: cmd_args, coverage_mode: GoCoverageMode, shared: bool = False) -> (Artifact, cmd_args): + cov_res = cover_srcs(ctx, pkg_name, coverage_mode, srcs, shared) + srcs = cov_res.srcs + coverage_vars = cov_res.variables + coverage_pkg = compile( + ctx, + pkg_name, + srcs = srcs, + deps = ctx.attrs.deps + ctx.attrs.exported_deps, + compile_flags = ctx.attrs.compiler_flags, + coverage_mode = coverage_mode, + shared = shared, + ) + return (coverage_pkg, coverage_vars) + def go_library_impl(ctx: AnalysisContext) -> list[Provider]: pkgs = {} default_output = None @@ -36,31 +52,24 @@ def go_library_impl(ctx: AnalysisContext) -> list[Provider]: # We need to set CGO_DESABLED for "pure" Go libraries, otherwise CGo files may be selected for compilation. srcs = get_filtered_srcs(ctx, ctx.attrs.srcs, force_disable_cgo = True) + shared = ctx.attrs._compile_shared - static_pkg = compile( + compiled_pkg = compile( ctx, pkg_name, srcs = srcs, deps = ctx.attrs.deps + ctx.attrs.exported_deps, compile_flags = ctx.attrs.compiler_flags, assemble_flags = ctx.attrs.assembler_flags, - shared = False, + shared = shared, ) - shared_pkg = compile( - ctx, - pkg_name, - srcs = srcs, - deps = ctx.attrs.deps + ctx.attrs.exported_deps, - compile_flags = ctx.attrs.compiler_flags, - assemble_flags = ctx.attrs.assembler_flags, - shared = True, - ) + pkg_with_coverage = {mode: _compile_with_coverage(ctx, pkg_name, srcs, mode, shared) for mode in GoCoverageMode} - default_output = static_pkg + default_output = compiled_pkg pkgs[pkg_name] = GoPkg( - shared = shared_pkg, - static = static_pkg, + pkg = compiled_pkg, + pkg_with_coverage = pkg_with_coverage, ) return [ diff --git a/prelude/go/go_stdlib.bzl b/prelude/go/go_stdlib.bzl new file mode 100644 index 0000000000000..f09ee8f2df657 --- /dev/null +++ b/prelude/go/go_stdlib.bzl @@ -0,0 +1,76 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load(":packages.bzl", "GoStdlib") +load(":toolchain.bzl", "GoToolchainInfo", "evaluate_cgo_enabled", "get_toolchain_cmd_args") + +def go_stdlib_impl(ctx: AnalysisContext) -> list[Provider]: + go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] + stdlib_pkgdir = ctx.actions.declare_output("stdlib_pkgdir", dir = True) + cgo_enabled = evaluate_cgo_enabled(go_toolchain, ctx.attrs._cgo_enabled) + tags = go_toolchain.tags + linker_flags = [] + go_toolchain.linker_flags + assembler_flags = [] + go_toolchain.assembler_flags + compiler_flags = [] + go_toolchain.compiler_flags + compiler_flags += ["-buildid="] # Make builds reproducible. + if ctx.attrs._compile_shared: + assembler_flags += ["-shared"] + compiler_flags += ["-shared"] + + go_wrapper_args = [] + cxx_toolchain = go_toolchain.cxx_toolchain_for_linking + if cxx_toolchain != None: + c_compiler = cxx_toolchain.c_compiler_info + + cgo_ldflags = cmd_args( + cxx_toolchain.linker_info.linker_flags, + go_toolchain.external_linker_flags, + ) + + go_wrapper_args += [ + cmd_args(c_compiler.compiler, format = "--cc={}").absolute_prefix("%cwd%/"), + cmd_args([c_compiler.compiler_flags, go_toolchain.c_compiler_flags], format = "--cgo_cflags={}").absolute_prefix("%cwd%/"), + cmd_args(c_compiler.preprocessor_flags, format = "--cgo_cppflags={}").absolute_prefix("%cwd%/"), + cmd_args(cgo_ldflags, format = "--cgo_ldflags={}").absolute_prefix("%cwd%/"), + ] + + cmd = get_toolchain_cmd_args(go_toolchain, go_root = True) + cmd.add([ + "GODEBUG={}".format("installgoroot=all"), + "CGO_ENABLED={}".format("1" if cgo_enabled else "0"), + go_toolchain.go_wrapper, + go_toolchain.go, + go_wrapper_args, + "install", + "-pkgdir", + stdlib_pkgdir.as_output(), + cmd_args(["-asmflags=", cmd_args(assembler_flags, delimiter = " ")], delimiter = "") if assembler_flags else [], + cmd_args(["-gcflags=", cmd_args(compiler_flags, delimiter = " ")], delimiter = "") if compiler_flags else [], + cmd_args(["-ldflags=", cmd_args(linker_flags, delimiter = " ")], delimiter = "") if linker_flags else [], + ["-tags", ",".join(tags)] if tags else [], + "std", + ]) + + ctx.actions.run(cmd, category = "go_build_stdlib", identifier = "go_build_stdlib") + + importcfg = ctx.actions.declare_output("stdlib.importcfg") + ctx.actions.run( + [ + go_toolchain.gen_stdlib_importcfg, + "--stdlib", + stdlib_pkgdir, + "--output", + importcfg.as_output(), + ], + category = "go_gen_stdlib_importcfg", + identifier = "go_gen_stdlib_importcfg", + ) + + return [ + DefaultInfo(default_output = stdlib_pkgdir), + GoStdlib(pkgdir = stdlib_pkgdir, importcfg = importcfg), + ] diff --git a/prelude/go/go_test.bzl b/prelude/go/go_test.bzl index 3ee295defe45b..f32d171822e7c 100644 --- a/prelude/go/go_test.bzl +++ b/prelude/go/go_test.bzl @@ -9,22 +9,26 @@ load( "@prelude//linking:link_info.bzl", "LinkStyle", ) +load( + "@prelude//tests:re_utils.bzl", + "get_re_executors_from_props", +) load( "@prelude//utils:utils.bzl", "map_val", "value_or", ) load("@prelude//test/inject_test_run_info.bzl", "inject_test_run_info") -load(":compile.bzl", "GoTestInfo", "compile", "get_filtered_srcs") +load(":compile.bzl", "GoTestInfo", "compile", "get_filtered_srcs", "get_inherited_compile_pkgs") load(":coverage.bzl", "GoCoverageMode", "cover_srcs") load(":link.bzl", "link") -load(":packages.bzl", "go_attr_pkg_name") +load(":packages.bzl", "go_attr_pkg_name", "pkg_artifact", "pkg_coverage_vars") def _gen_test_main( ctx: AnalysisContext, pkg_name: str, coverage_mode: [GoCoverageMode, None], - coverage_vars: [cmd_args, None], + coverage_vars: dict[str, cmd_args], srcs: cmd_args) -> Artifact: """ Generate a `main.go` which calls tests from the given sources. @@ -38,12 +42,15 @@ def _gen_test_main( cmd.add(cmd_args(pkg_name, format = "--import-path={}")) if coverage_mode != None: cmd.add("--cover-mode", coverage_mode.value) - if coverage_vars != None: - cmd.add(coverage_vars) + for _, vars in coverage_vars.items(): + cmd.add(vars) cmd.add(srcs) ctx.actions.run(cmd, category = "go_test_main_gen") return output +def is_subpackage_of(other_pkg_name: str, pkg_name: str) -> bool: + return pkg_name == other_pkg_name or other_pkg_name.startswith(pkg_name + "/") + def go_test_impl(ctx: AnalysisContext) -> list[Provider]: deps = ctx.attrs.deps srcs = ctx.attrs.srcs @@ -64,12 +71,21 @@ def go_test_impl(ctx: AnalysisContext) -> list[Provider]: # If coverage is enabled for this test, we need to preprocess the sources # with the Go cover tool. coverage_mode = None - coverage_vars = None + coverage_vars = {} + pkgs = {} if ctx.attrs.coverage_mode != None: coverage_mode = GoCoverageMode(ctx.attrs.coverage_mode) - cov_res = cover_srcs(ctx, pkg_name, coverage_mode, srcs) + cov_res = cover_srcs(ctx, pkg_name, coverage_mode, srcs, False) srcs = cov_res.srcs - coverage_vars = cov_res.variables + coverage_vars[pkg_name] = cov_res.variables + + # Get all packages that are linked to the test (i.e. the entire dependency tree) + for name, pkg in get_inherited_compile_pkgs(deps).items(): + if ctx.label != None and is_subpackage_of(name, ctx.label.package): + artifact = pkg_artifact(pkg, coverage_mode) + vars = pkg_coverage_vars("", pkg, coverage_mode) + coverage_vars[name] = vars + pkgs[name] = artifact # Compile all tests into a package. tests = compile( @@ -77,22 +93,27 @@ def go_test_impl(ctx: AnalysisContext) -> list[Provider]: pkg_name, srcs, deps = deps, + pkgs = pkgs, compile_flags = ctx.attrs.compiler_flags, + coverage_mode = coverage_mode, ) # Generate a main function which runs the tests and build that into another # package. gen_main = _gen_test_main(ctx, pkg_name, coverage_mode, coverage_vars, srcs) - main = compile(ctx, "main", cmd_args(gen_main), pkgs = {pkg_name: tests}) + pkgs[pkg_name] = tests + main = compile(ctx, "main", cmd_args(gen_main), pkgs = pkgs, coverage_mode = coverage_mode) # Link the above into a Go binary. (bin, runtime_files, external_debug_info) = link( ctx = ctx, main = main, - pkgs = {pkg_name: tests}, + pkgs = pkgs, deps = deps, link_style = value_or(map_val(LinkStyle, ctx.attrs.link_style), LinkStyle("static")), linker_flags = ctx.attrs.linker_flags, + shared = False, + coverage_mode = coverage_mode, ) run_cmd = cmd_args(bin).hidden(runtime_files, external_debug_info) @@ -101,6 +122,9 @@ def go_test_impl(ctx: AnalysisContext) -> list[Provider]: for resource in ctx.attrs.resources: run_cmd.hidden(ctx.actions.copy_file(resource.short_path, resource)) + # Setup RE executors based on the `remote_execution` param. + re_executor, executor_overrides = get_re_executors_from_props(ctx) + return inject_test_run_info( ctx, ExternalRunnerTestInfo( @@ -109,8 +133,11 @@ def go_test_impl(ctx: AnalysisContext) -> list[Provider]: env = ctx.attrs.env, labels = ctx.attrs.labels, contacts = ctx.attrs.contacts, + default_executor = re_executor, + executor_overrides = executor_overrides, # FIXME: Consider setting to true - run_from_project_root = False, + run_from_project_root = re_executor != None, + use_project_relative_paths = re_executor != None, ), ) + [ DefaultInfo( diff --git a/prelude/go/link.bzl b/prelude/go/link.bzl index bb7957f995f03..09d637edf3b36 100644 --- a/prelude/go/link.bzl +++ b/prelude/go/link.bzl @@ -29,12 +29,17 @@ load( "@prelude//utils:utils.bzl", "map_idx", ) + +# @unused this comment is to make the linter happy. The linter thinks +# GoCoverageMode is unused despite it being used in the function signature of +# link. +load(":coverage.bzl", "GoCoverageMode") load( ":packages.bzl", "GoPkg", # @Unused used as type + "make_importcfg", "merge_pkgs", "pkg_artifacts", - "stdlib_pkg_artifacts", ) load(":toolchain.bzl", "GoToolchainInfo", "get_toolchain_cmd_args") @@ -92,7 +97,7 @@ def _process_shared_dependencies( shared_libs[name] = shared_lib.lib return executable_shared_lib_arguments( - ctx.actions, + ctx, ctx.attrs._go_toolchain[GoToolchainInfo].cxx_toolchain_for_linking, artifact, shared_libs, @@ -108,7 +113,8 @@ def link( link_style: LinkStyle = LinkStyle("static"), linker_flags: list[typing.Any] = [], external_linker_flags: list[typing.Any] = [], - shared: bool = False): + shared: bool = False, + coverage_mode: [GoCoverageMode, None] = None): go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] if go_toolchain.env_go_os == "windows": executable_extension = ".exe" @@ -122,10 +128,7 @@ def link( cmd = get_toolchain_cmd_args(go_toolchain) cmd.add(go_toolchain.linker) - if shared: - cmd.add(go_toolchain.linker_flags_shared) - else: - cmd.add(go_toolchain.linker_flags_static) + cmd.add(go_toolchain.linker_flags) cmd.add("-o", output.as_output()) cmd.add("-buildmode=" + _build_mode_param(build_mode)) @@ -134,20 +137,12 @@ def link( # Add inherited Go pkgs to library search path. all_pkgs = merge_pkgs([ pkgs, - pkg_artifacts(get_inherited_link_pkgs(deps), shared = shared), - stdlib_pkg_artifacts(go_toolchain, shared = shared), + pkg_artifacts(get_inherited_link_pkgs(deps), coverage_mode = coverage_mode), ]) - importcfg_content = [] - for name_, pkg_ in all_pkgs.items(): - # Hack: we use cmd_args get "artifact" valid path and write it to a file. - importcfg_content.append(cmd_args("packagefile ", name_, "=", pkg_, delimiter = "")) - - importcfg = ctx.actions.declare_output("importcfg") - ctx.actions.write(importcfg.as_output(), importcfg_content) + importcfg = make_importcfg(ctx, "", "", all_pkgs, with_importmap = False) cmd.add("-importcfg", importcfg) - cmd.hidden(all_pkgs.values()) executable_args = _process_shared_dependencies(ctx, output, deps, link_style) @@ -186,8 +181,6 @@ def link( cxx_link_cmd = cmd_args( [ cxx_toolchain.linker_info.linker, - cxx_toolchain.linker_info.linker_flags, - go_toolchain.external_linker_flags, ext_link_args, "%*" if is_win else "\"$@\"", ], @@ -200,6 +193,11 @@ def link( is_executable = True, ) cmd.add("-extld", linker_wrapper).hidden(cxx_link_cmd) + cmd.add("-extldflags", cmd_args( + cxx_toolchain.linker_info.linker_flags, + go_toolchain.external_linker_flags, + delimiter = " ", + )) cmd.add(linker_flags) diff --git a/prelude/go/packages.bzl b/prelude/go/packages.bzl index 26e90b548bed7..6ca43871f9a0e 100644 --- a/prelude/go/packages.bzl +++ b/prelude/go/packages.bzl @@ -5,16 +5,21 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//:artifacts.bzl", "ArtifactGroupInfo") load("@prelude//go:toolchain.bzl", "GoToolchainInfo") load("@prelude//utils:utils.bzl", "value_or") +load(":coverage.bzl", "GoCoverageMode") GoPkg = record( - # Built w/ `-shared`. - shared = field(Artifact), - # Built w/o `-shared`. - static = field(Artifact), cgo = field(bool, default = False), + pkg = field(Artifact), + pkg_with_coverage = field(dict[GoCoverageMode, (Artifact, cmd_args)]), +) + +GoStdlib = provider( + fields = { + "importcfg": provider_field(Artifact), + "pkgdir": provider_field(Artifact), + }, ) def go_attr_pkg_name(ctx: AnalysisContext) -> str: @@ -39,35 +44,62 @@ def merge_pkgs(pkgss: list[dict[str, typing.Any]]) -> dict[str, typing.Any]: return all_pkgs -def pkg_artifacts(pkgs: dict[str, GoPkg], shared: bool = False) -> dict[str, Artifact]: +def pkg_artifact(pkg: GoPkg, coverage_mode: [GoCoverageMode, None]) -> Artifact: + if coverage_mode: + artifact = pkg.pkg_with_coverage + return artifact[coverage_mode][0] + return pkg.pkg + +def pkg_coverage_vars(name: str, pkg: GoPkg, coverage_mode: [GoCoverageMode, None]) -> [cmd_args, None]: + if coverage_mode: + artifact = pkg.pkg_with_coverage + if coverage_mode not in artifact: + fail("coverage variables don't exist for {}".format(name)) + return artifact[coverage_mode][1] + fail("coverage variables were requested but coverage_mode is None") + +def pkg_artifacts(pkgs: dict[str, GoPkg], coverage_mode: [GoCoverageMode, None] = None) -> dict[str, Artifact]: """ Return a map package name to a `shared` or `static` package artifact. """ return { - name: pkg.shared if shared else pkg.static + name: pkg_artifact(pkg, coverage_mode) for name, pkg in pkgs.items() } -def stdlib_pkg_artifacts(toolchain: GoToolchainInfo, shared: bool = False) -> dict[str, Artifact]: - """ - Return a map package name to a `shared` or `static` package artifact of stdlib. - """ +def make_importcfg( + ctx: AnalysisContext, + root: str, + pkg_name: str, + own_pkgs: dict[str, typing.Any], + with_importmap: bool) -> cmd_args: + go_toolchain = ctx.attrs._go_toolchain[GoToolchainInfo] + stdlib = ctx.attrs._go_stdlib[GoStdlib] - prebuilt_stdlib = toolchain.prebuilt_stdlib_shared if shared else toolchain.prebuilt_stdlib - stdlib_pkgs = prebuilt_stdlib[ArtifactGroupInfo].artifacts + content = [] + for name_, pkg_ in own_pkgs.items(): + # Hack: we use cmd_args get "artifact" valid path and write it to a file. + content.append(cmd_args("packagefile ", name_, "=", pkg_, delimiter = "")) - if len(stdlib_pkgs) == 0: - fail("Stdlib for current platfrom is missing from toolchain.") + # Future work: support importmap in buck rules insted of hacking here. + if with_importmap and name_.startswith("third-party-source/go/"): + real_name_ = name_.removeprefix("third-party-source/go/") + content.append(cmd_args("importmap ", real_name_, "=", name_, delimiter = "")) - pkgs = {} - for pkg in stdlib_pkgs: - # remove first directory like `pgk` - _, _, temp_path = pkg.short_path.partition("/") + own_importcfg = ctx.actions.declare_output(root, "{}.importcfg".format(pkg_name)) + ctx.actions.write(own_importcfg, content) - # remove second directory like `darwin_amd64` - # now we have name like `net/http.a` - _, _, pkg_relpath = temp_path.partition("/") - name = pkg_relpath.removesuffix(".a") # like `net/http` - pkgs[name] = pkg + final_importcfg = ctx.actions.declare_output(root, "{}.final.importcfg".format(pkg_name)) + ctx.actions.run( + [ + go_toolchain.concat_files, + "--output", + final_importcfg.as_output(), + stdlib.importcfg, + own_importcfg, + ], + category = "concat_importcfgs", + identifier = "{}/{}".format(root, pkg_name), + ) - return pkgs + return cmd_args(final_importcfg).hidden(stdlib.pkgdir).hidden(own_pkgs.values()) diff --git a/prelude/go/toolchain.bzl b/prelude/go/toolchain.bzl index 3f00630e3b688..2311a2fb0f92c 100644 --- a/prelude/go/toolchain.bzl +++ b/prelude/go/toolchain.bzl @@ -9,12 +9,16 @@ GoToolchainInfo = provider( # @unsorted-dict-items fields = { "assembler": provider_field(typing.Any, default = None), + "assembler_flags": provider_field(typing.Any, default = None), + "c_compiler_flags": provider_field(typing.Any, default = None), "cgo": provider_field(typing.Any, default = None), "cgo_wrapper": provider_field(typing.Any, default = None), + "gen_stdlib_importcfg": provider_field(typing.Any, default = None), + "go_wrapper": provider_field(typing.Any, default = None), "compile_wrapper": provider_field(typing.Any, default = None), "compiler": provider_field(typing.Any, default = None), - "compiler_flags_shared": provider_field(typing.Any, default = None), - "compiler_flags_static": provider_field(typing.Any, default = None), + "compiler_flags": provider_field(typing.Any, default = None), + "concat_files": provider_field(typing.Any, default = None), "cover": provider_field(typing.Any, default = None), "cover_srcs": provider_field(typing.Any, default = None), "cxx_toolchain_for_linking": provider_field(typing.Any, default = None), @@ -26,11 +30,11 @@ GoToolchainInfo = provider( "filter_srcs": provider_field(typing.Any, default = None), "go": provider_field(typing.Any, default = None), "linker": provider_field(typing.Any, default = None), - "linker_flags_shared": provider_field(typing.Any, default = None), - "linker_flags_static": provider_field(typing.Any, default = None), + "linker_flags": provider_field(typing.Any, default = None), "packer": provider_field(typing.Any, default = None), "prebuilt_stdlib": provider_field(typing.Any, default = None), "prebuilt_stdlib_shared": provider_field(typing.Any, default = None), + "prebuilt_stdlib_noncgo": provider_field(typing.Any, default = None), "tags": provider_field(typing.Any, default = None), }, ) @@ -60,3 +64,16 @@ def get_toolchain_cmd_args(toolchain: GoToolchainInfo, go_root = True, force_dis cmd.add("CGO_ENABLED=1") return cmd + +# Sets default value of cgo_enabled attribute based on the presence of C++ toolchain. +def evaluate_cgo_enabled(toolchain: GoToolchainInfo, cgo_enabled: [bool, None]) -> bool: + cxx_toolchain_available = toolchain.cxx_toolchain_for_linking != None + + if cgo_enabled and not cxx_toolchain_available: + fail("Cgo requires a C++ toolchain. Set cgo_enabled=None|False.") + + if cgo_enabled != None: + return cgo_enabled + + # Return True if cxx_toolchain availabe for current configuration, otherwiese to False. + return cxx_toolchain_available diff --git a/prelude/go/tools/BUCK.v2 b/prelude/go/tools/BUCK.v2 index b7499e98f9444..92b006f5bfce4 100644 --- a/prelude/go/tools/BUCK.v2 +++ b/prelude/go/tools/BUCK.v2 @@ -6,6 +6,12 @@ prelude.python_bootstrap_binary( visibility = ["PUBLIC"], ) +prelude.python_bootstrap_binary( + name = "concat_files", + main = "concat_files.py", + visibility = ["PUBLIC"], +) + prelude.python_bootstrap_binary( name = "cover_srcs", main = "cover_srcs.py", @@ -24,6 +30,18 @@ prelude.python_bootstrap_binary( visibility = ["PUBLIC"], ) +prelude.python_bootstrap_binary( + name = "gen_stdlib_importcfg", + main = "gen_stdlib_importcfg.py", + visibility = ["PUBLIC"], +) + +prelude.python_bootstrap_binary( + name = "go_wrapper", + main = "go_wrapper.py", + visibility = ["PUBLIC"], +) + prelude.go_binary( name = "testmaingen", srcs = [ @@ -33,3 +51,8 @@ prelude.go_binary( "PUBLIC", ], ) + +prelude.go_stdlib( + name = "stdlib", + visibility = ["PUBLIC"], +) diff --git a/prelude/go/tools/concat_files.py b/prelude/go/tools/concat_files.py new file mode 100644 index 0000000000000..145335a288597 --- /dev/null +++ b/prelude/go/tools/concat_files.py @@ -0,0 +1,33 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import argparse +import sys +from pathlib import Path + + +def main(argv): + parser = argparse.ArgumentParser(fromfile_prefix_chars="@") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("files", type=Path, nargs="*") + args = parser.parse_args(argv[1:]) + + if len(args.files) == 0: + print( + "usage: concat_files.py --output out.txt in1.txt in2.txt", file=sys.stderr + ) + return 1 + + with open(args.output, "wb") as outfile: + for f in args.files: + with open(f, "rb") as infile: + outfile.write(infile.read()) + + return 0 + + +sys.exit(main(sys.argv)) diff --git a/prelude/go/tools/cover_srcs.py b/prelude/go/tools/cover_srcs.py index 4dcaf2cc51b79..1dabf647ad9ae 100644 --- a/prelude/go/tools/cover_srcs.py +++ b/prelude/go/tools/cover_srcs.py @@ -40,7 +40,8 @@ def main(argv): args.covered_srcs_dir.mkdir(parents=True) for src in args.srcs: - if src.name.endswith("_test.go"): + # don't cover test files or non-go files (e.g. assembly files) + if src.name.endswith("_test.go") or not src.name.endswith(".go"): out_srcs.append(src) else: var = _var(args.pkg_name, src) diff --git a/prelude/go/tools/gen_stdlib_importcfg.py b/prelude/go/tools/gen_stdlib_importcfg.py new file mode 100644 index 0000000000000..ce973c0ab7df3 --- /dev/null +++ b/prelude/go/tools/gen_stdlib_importcfg.py @@ -0,0 +1,32 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import argparse +import os +import sys +from pathlib import Path + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--stdlib", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + + args = parser.parse_args() + + with open(args.output, "w") as f: + for root, _dirs, files in os.walk(args.stdlib): + for file in files: + pkg_path = Path(root, file) + pkg_name, _ = os.path.splitext(pkg_path.relative_to(args.stdlib)) + # package names always use unix slashes + pkg_name = pkg_name.replace("\\", "/") + f.write(f"packagefile {pkg_name}={pkg_path}\n") + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/prelude/go/tools/go_wrapper.py b/prelude/go/tools/go_wrapper.py new file mode 100644 index 0000000000000..bb830da9773b7 --- /dev/null +++ b/prelude/go/tools/go_wrapper.py @@ -0,0 +1,60 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def main(argv): + """ + This is a wrapper script around the `go` binary. + - It fixes GOROOT and GOCACHE + """ + if len(argv) < 2: + print("usage: go_wrapper.py ", file=sys.stderr) + return 1 + + wrapped_binary = Path(argv[1]) + + parser = argparse.ArgumentParser() + parser.add_argument("--cc", action="append", default=[]) + parser.add_argument("--cgo_cflags", action="append", default=[]) + parser.add_argument("--cgo_cppflags", action="append", default=[]) + parser.add_argument("--cgo_ldflags", action="append", default=[]) + parsed, unknown = parser.parse_known_args(argv[2:]) + + env = os.environ.copy() + # Make paths absolute, otherwise go build will fail. + env["GOROOT"] = os.path.realpath(env["GOROOT"]) + env["GOCACHE"] = os.path.realpath(env["BUCK_SCRATCH_PATH"]) + + cwd = os.getcwd() + if len(parsed.cc) > 0: + env["CC"] = " ".join([arg.replace("%cwd%", cwd) for arg in parsed.cc]) + + if len(parsed.cgo_cflags) > 0: + env["CGO_CFLAGS"] = " ".join( + [arg.replace("%cwd%", cwd) for arg in parsed.cgo_cflags] + ) + + if len(parsed.cgo_cppflags) > 0: + env["CGO_CPPFLAGS"] = " ".join( + [arg.replace("%cwd%", cwd) for arg in parsed.cgo_cppflags] + ) + + if len(parsed.cgo_ldflags) > 0: + env["CGO_LDFLAGS"] = " ".join( + [arg.replace("%cwd%", cwd) for arg in parsed.cgo_ldflags] + ) + + return subprocess.call([wrapped_binary] + unknown, env=env) + + +sys.exit(main(sys.argv)) diff --git a/prelude/go/transitions/defs.bzl b/prelude/go/transitions/defs.bzl new file mode 100644 index 0000000000000..5686852ac4902 --- /dev/null +++ b/prelude/go/transitions/defs.bzl @@ -0,0 +1,103 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +def _cgo_enabled_transition(platform, refs, attrs): + constraints = platform.configuration.constraints + + # Cancel transition if the value already set + # to enable using configuration modifiers for overiding this option + cgo_enabled_setting = refs.cgo_enabled_auto[ConstraintValueInfo].setting + if cgo_enabled_setting.label in constraints: + return platform + + if attrs.cgo_enabled == None: + cgo_enabled_ref = refs.cgo_enabled_auto + elif attrs.cgo_enabled == True: + cgo_enabled_ref = refs.cgo_enabled_true + else: + cgo_enabled_ref = refs.cgo_enabled_false + + cgo_enabled_value = cgo_enabled_ref[ConstraintValueInfo] + constraints[cgo_enabled_value.setting.label] = cgo_enabled_value + + new_cfg = ConfigurationInfo( + constraints = constraints, + values = platform.configuration.values, + ) + + return PlatformInfo( + label = platform.label, + configuration = new_cfg, + ) + +def _compile_shared_transition(platform, refs, _): + compile_shared_value = refs.compile_shared_value[ConstraintValueInfo] + constraints = platform.configuration.constraints + constraints[compile_shared_value.setting.label] = compile_shared_value + new_cfg = ConfigurationInfo( + constraints = constraints, + values = platform.configuration.values, + ) + + return PlatformInfo( + label = platform.label, + configuration = new_cfg, + ) + +def _chain_transitions(transitions): + def tr(platform, refs, attrs): + for t in transitions: + platform = t(platform, refs, attrs) + return platform + + return tr + +go_binary_transition = transition( + impl = _chain_transitions([_cgo_enabled_transition, _compile_shared_transition]), + refs = { + "cgo_enabled_auto": "prelude//go/constraints:cgo_enabled_auto", + "cgo_enabled_false": "prelude//go/constraints:cgo_enabled_false", + "cgo_enabled_true": "prelude//go/constraints:cgo_enabled_true", + "compile_shared_value": "prelude//go/constraints:compile_shared_false", + }, + attrs = ["cgo_enabled"], +) + +go_test_transition = transition( + impl = _chain_transitions([_cgo_enabled_transition, _compile_shared_transition]), + refs = { + "cgo_enabled_auto": "prelude//go/constraints:cgo_enabled_auto", + "cgo_enabled_false": "prelude//go/constraints:cgo_enabled_false", + "cgo_enabled_true": "prelude//go/constraints:cgo_enabled_true", + "compile_shared_value": "prelude//go/constraints:compile_shared_false", + }, + attrs = ["cgo_enabled"], +) + +go_exported_library_transition = transition( + impl = _chain_transitions([_cgo_enabled_transition, _compile_shared_transition]), + refs = { + "cgo_enabled_auto": "prelude//go/constraints:cgo_enabled_auto", + "cgo_enabled_false": "prelude//go/constraints:cgo_enabled_false", + "cgo_enabled_true": "prelude//go/constraints:cgo_enabled_true", + "compile_shared_value": "prelude//go/constraints:compile_shared_true", + }, + attrs = ["cgo_enabled"], +) + +cgo_enabled_attr = attrs.default_only(attrs.option(attrs.bool(), default = select({ + "DEFAULT": None, + "prelude//go/constraints:cgo_enabled_auto": None, + "prelude//go/constraints:cgo_enabled_false": False, + "prelude//go/constraints:cgo_enabled_true": True, +}))) + +compile_shared_attr = attrs.default_only(attrs.bool(default = select({ + "DEFAULT": False, + "prelude//go/constraints:compile_shared_false": False, + "prelude//go/constraints:compile_shared_true": True, +}))) diff --git a/prelude/haskell/compile.bzl b/prelude/haskell/compile.bzl index 8fc9f6aeb62a3..3cfde48b421d5 100644 --- a/prelude/haskell/compile.bzl +++ b/prelude/haskell/compile.bzl @@ -10,9 +10,12 @@ load( "cxx_inherited_preprocessor_infos", "cxx_merge_cpreprocessors", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", +) load( "@prelude//haskell:link_info.bzl", - "HaskellLinkInfo", "merge_haskell_link_infos", ) load( @@ -21,7 +24,8 @@ load( ) load( "@prelude//haskell:util.bzl", - "attr_deps", + "attr_deps_haskell_lib_infos", + "attr_deps_haskell_link_infos", "get_artifact_suffix", "is_haskell_src", "output_extensions", @@ -47,72 +51,12 @@ CompileArgsInfo = record( args_for_file = field(cmd_args), ) -# If the target is a haskell library, the HaskellLibraryProvider -# contains its HaskellLibraryInfo. (in contrast to a HaskellLinkInfo, -# which contains the HaskellLibraryInfo for all the transitive -# dependencies). Direct dependencies are treated differently from -# indirect dependencies for the purposes of module visibility. -HaskellLibraryProvider = provider( - fields = { - "lib": provider_field(typing.Any, default = None), # dict[LinkStyle, HaskellLibraryInfo] - "prof_lib": provider_field(typing.Any, default = None), # dict[LinkStyle, HaskellLibraryInfo] - }, -) - -# A record of a Haskell library. -HaskellLibraryInfo = record( - # The library target name: e.g. "rts" - name = str, - # package config database: e.g. platform009/build/ghc/lib/package.conf.d - db = Artifact, - # e.g. "base-4.13.0.0" - id = str, - # Import dirs indexed by profiling enabled/disabled - import_dirs = dict[bool, Artifact], - stub_dirs = list[Artifact], - - # This field is only used as hidden inputs to compilation, to - # support Template Haskell which may need access to the libraries - # at compile time. The real library flags are propagated up the - # dependency graph via MergedLinkInfo. - libs = field(list[Artifact], []), - # Package version, used to specify the full package when exposing it, - # e.g. filepath-1.4.2.1, deepseq-1.4.4.0. - # Internal packages default to 1.0.0, e.g. `fbcode-dsi-logger-hs-types-1.0.0`. - version = str, - is_prebuilt = bool, - profiling_enabled = bool, -) - PackagesInfo = record( exposed_package_args = cmd_args, packagedb_args = cmd_args, transitive_deps = field(list[HaskellLibraryInfo]), ) -def _attr_deps_haskell_link_infos(ctx: AnalysisContext) -> list[HaskellLinkInfo]: - return filter( - None, - [ - d.get(HaskellLinkInfo) - for d in attr_deps(ctx) + ctx.attrs.template_deps - ], - ) - -def _attr_deps_haskell_lib_infos( - ctx: AnalysisContext, - link_style: LinkStyle, - enable_profiling: bool) -> list[HaskellLibraryInfo]: - if enable_profiling and link_style == LinkStyle("shared"): - fail("Profiling isn't supported when using dynamic linking") - return [ - x.prof_lib[link_style] if enable_profiling else x.lib[link_style] - for x in filter(None, [ - d.get(HaskellLibraryProvider) - for d in attr_deps(ctx) + ctx.attrs.template_deps - ]) - ] - def _package_flag(toolchain: HaskellToolchainInfo) -> str: if toolchain.support_expose_package: return "-expose-package" @@ -130,7 +74,7 @@ def get_packages_info( # particular order and we really want to remove duplicates (there # are a *lot* of duplicates). libs = {} - direct_deps_link_info = _attr_deps_haskell_link_infos(ctx) + direct_deps_link_info = attr_deps_haskell_link_infos(ctx) merged_hs_link_info = merge_haskell_link_infos(direct_deps_link_info) hs_link_info = merged_hs_link_info.prof_info if enable_profiling else merged_hs_link_info.info @@ -161,7 +105,7 @@ def get_packages_info( # direct and transitive (e.g. `fbcode-common-hs-util-hs-array`) packagedb_args.add("-package-db", lib.db) - haskell_direct_deps_lib_infos = _attr_deps_haskell_lib_infos( + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( ctx, link_style, enable_profiling, diff --git a/prelude/haskell/haskell.bzl b/prelude/haskell/haskell.bzl index b35edd60f9f2d..47d453da34e80 100644 --- a/prelude/haskell/haskell.bzl +++ b/prelude/haskell/haskell.bzl @@ -30,6 +30,11 @@ load( "get_transitive_deps_matching_labels", "is_link_group_shlib", ) +load( + "@prelude//cxx:linker.bzl", + "LINKERS", + "get_shared_library_flags", +) load( "@prelude//cxx:preprocessor.bzl", "CPreprocessor", @@ -40,17 +45,21 @@ load( load( "@prelude//haskell:compile.bzl", "CompileResultInfo", - "HaskellLibraryInfo", - "HaskellLibraryProvider", "compile", ) load( "@prelude//haskell:haskell_haddock.bzl", "haskell_haddock_lib", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", + "HaskellLibraryProvider", +) load( "@prelude//haskell:link_info.bzl", "HaskellLinkInfo", + "HaskellProfLinkInfo", "attr_link_style", "cxx_toolchain_link_style", "merge_haskell_link_infos", @@ -62,6 +71,10 @@ load( load( "@prelude//haskell:util.bzl", "attr_deps", + "attr_deps_haskell_link_infos_sans_template_deps", + "attr_deps_merged_link_infos", + "attr_deps_profiling_link_infos", + "attr_deps_shared_library_infos", "get_artifact_suffix", "is_haskell_src", "output_extensions", @@ -130,21 +143,6 @@ HaskellIndexInfo = provider( }, ) -# HaskellProfLinkInfo exposes the MergedLinkInfo of a target and all of its -# dependencies built for profiling. This allows top-level targets (e.g. -# `haskell_binary`) to be defined with profiling enabled by default. -HaskellProfLinkInfo = provider( - fields = { - "prof_infos": provider_field(typing.Any, default = None), # MergedLinkInfo - }, -) - -# -- - -# Disable until we have a need to call this. -# def _attr_deps_merged_link_infos(ctx: AnalysisContext) -> [MergedLinkInfo]: -# return filter(None, [d[MergedLinkInfo] for d in attr_deps(ctx)]) - # This conversion is non-standard, see TODO about link style below def _to_lib_output_style(link_style: LinkStyle) -> LibOutputStyle: return default_output_style_for_link_strategy(to_link_strategy(link_style)) @@ -479,6 +477,17 @@ HaskellLibBuildOutput = record( libs = list[Artifact], ) +def _get_haskell_shared_library_name_linker_flags(linker_type: str, soname: str) -> list[str]: + if linker_type == "gnu": + return ["-Wl,-soname,{}".format(soname)] + elif linker_type == "darwin": + # Passing `-install_name @rpath/...` or + # `-Xlinker -install_name -Xlinker @rpath/...` instead causes + # ghc-9.6.3: panic! (the 'impossible' happened) + return ["-Wl,-install_name,@rpath/{}".format(soname)] + else: + fail("Unknown linker type '{}'.".format(linker_type)) + def _build_haskell_lib( ctx, libname: str, @@ -511,8 +520,9 @@ def _build_haskell_lib( if link_style == LinkStyle("static_pic"): libstem += "_pic" + dynamic_lib_suffix = "." + LINKERS[linker_info.type].default_shared_library_extension static_lib_suffix = "_p.a" if enable_profiling else ".a" - libfile = "lib" + libstem + (".so" if link_style == LinkStyle("shared") else static_lib_suffix) + libfile = "lib" + libstem + (dynamic_lib_suffix if link_style == LinkStyle("shared") else static_lib_suffix) lib_short_path = paths.join("lib-{}".format(artifact_suffix), libfile) @@ -528,12 +538,12 @@ def _build_haskell_lib( link.add(ctx.attrs.linker_flags) link.add("-o", lib.as_output()) link.add( - "-shared", + get_shared_library_flags(linker_info.type), "-dynamic", - "-optl", - "-Wl,-soname", - "-optl", - "-Wl," + libfile, + cmd_args( + _get_haskell_shared_library_name_linker_flags(linker_info.type, libfile), + prepend = "-optl", + ), ) link.add(objfiles) @@ -635,27 +645,10 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: preferred_linkage = Linkage("static") # Get haskell and native link infos from all deps - hlis = [] - nlis = [] - prof_nlis = [] - shared_library_infos = [] - for lib in attr_deps(ctx): - li = lib.get(HaskellLinkInfo) - if li != None: - hlis.append(li) - li = lib.get(MergedLinkInfo) - if li != None: - nlis.append(li) - if HaskellLinkInfo not in lib: - # MergedLinkInfo from non-haskell deps should be part of the - # profiling MergedLinkInfo - prof_nlis.append(li) - li = lib.get(HaskellProfLinkInfo) - if li != None: - prof_nlis.append(li.prof_infos) - li = lib.get(SharedLibraryInfo) - if li != None: - shared_library_infos.append(li) + hlis = attr_deps_haskell_link_infos_sans_template_deps(ctx) + nlis = attr_deps_merged_link_infos(ctx) + prof_nlis = attr_deps_profiling_link_infos(ctx) + shared_library_infos = attr_deps_shared_library_infos(ctx) solibs = {} link_infos = {} diff --git a/prelude/haskell/haskell_ghci.bzl b/prelude/haskell/haskell_ghci.bzl index a8b8c56ba9373..c15efdb14e98b 100644 --- a/prelude/haskell/haskell_ghci.bzl +++ b/prelude/haskell/haskell_ghci.bzl @@ -18,11 +18,14 @@ load( ) load( "@prelude//haskell:compile.bzl", - "HaskellLibraryInfo", - "HaskellLibraryProvider", "PackagesInfo", "get_packages_info", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", + "HaskellLibraryProvider", +) load( "@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", @@ -630,11 +633,11 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: package_symlinks_root = ctx.label.name + ".packages" packagedb_args = cmd_args(delimiter = " ") - prebuilt_packagedb_args = cmd_args(delimiter = " ") + prebuilt_packagedb_args_set = {} for lib in packages_info.transitive_deps: if lib.is_prebuilt: - prebuilt_packagedb_args.add(lib.db) + prebuilt_packagedb_args_set[lib.db] = lib.db else: lib_symlinks_root = paths.join( package_symlinks_root, @@ -664,6 +667,7 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: "packagedb", ), ) + prebuilt_packagedb_args = cmd_args(prebuilt_packagedb_args_set.values(), delimiter = " ") script_templates = [] for script_template in ctx.attrs.extra_script_templates: diff --git a/prelude/haskell/ide/README.md b/prelude/haskell/ide/README.md index c7867a541f2c3..4e58eed4f0a96 100644 --- a/prelude/haskell/ide/README.md +++ b/prelude/haskell/ide/README.md @@ -1,13 +1,14 @@ # Haskell Language Server integration This integration allows loading `haskell_binary` and `haskell_library` targets -on Haskell Language Server. This is accomplished via a BXL script that is -used to drive a hie-bios "bios" cradle. +on Haskell Language Server. This is accomplished via a BXL script that is used +to drive a hie-bios "bios" cradle. # Usage To print the list of GHC flags and targets for a Haskell source file: - buck2 bxl prelude//haskell/ide/ide.bxl -- --bios true --file +buck2 bxl prelude//haskell/ide/ide.bxl -- --bios true --file + To integrate with hie_bios, copy `hie.yaml` to your repo root diff --git a/prelude/haskell/ide/ide.bxl b/prelude/haskell/ide/ide.bxl index 9189073e81e53..425f75493fd47 100644 --- a/prelude/haskell/ide/ide.bxl +++ b/prelude/haskell/ide/ide.bxl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//haskell:compile.bzl", "HaskellLibraryProvider") +load("@prelude//haskell:library_info.bzl", "HaskellLibraryProvider") load("@prelude//haskell:link_info.bzl", "HaskellLinkInfo") load("@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo") load("@prelude//linking:link_info.bzl", "LinkStyle") diff --git a/prelude/haskell/library_info.bzl b/prelude/haskell/library_info.bzl new file mode 100644 index 0000000000000..028496e7b5208 --- /dev/null +++ b/prelude/haskell/library_info.bzl @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +# If the target is a haskell library, the HaskellLibraryProvider +# contains its HaskellLibraryInfo. (in contrast to a HaskellLinkInfo, +# which contains the HaskellLibraryInfo for all the transitive +# dependencies). Direct dependencies are treated differently from +# indirect dependencies for the purposes of module visibility. +HaskellLibraryProvider = provider( + fields = { + "lib": provider_field(typing.Any, default = None), # dict[LinkStyle, HaskellLibraryInfo] + "prof_lib": provider_field(typing.Any, default = None), # dict[LinkStyle, HaskellLibraryInfo] + }, +) + +# A record of a Haskell library. +HaskellLibraryInfo = record( + # The library target name: e.g. "rts" + name = str, + # package config database: e.g. platform009/build/ghc/lib/package.conf.d + db = Artifact, + # e.g. "base-4.13.0.0" + id = str, + # Import dirs indexed by profiling enabled/disabled + import_dirs = dict[bool, Artifact], + stub_dirs = list[Artifact], + + # This field is only used as hidden inputs to compilation, to + # support Template Haskell which may need access to the libraries + # at compile time. The real library flags are propagated up the + # dependency graph via MergedLinkInfo. + libs = field(list[Artifact], []), + # Package version, used to specify the full package when exposing it, + # e.g. filepath-1.4.2.1, deepseq-1.4.4.0. + # Internal packages default to 1.0.0, e.g. `fbcode-dsi-logger-hs-types-1.0.0`. + version = str, + is_prebuilt = bool, + profiling_enabled = bool, +) diff --git a/prelude/haskell/link_info.bzl b/prelude/haskell/link_info.bzl index 5bce615ba52c0..8699a875e09eb 100644 --- a/prelude/haskell/link_info.bzl +++ b/prelude/haskell/link_info.bzl @@ -23,6 +23,15 @@ HaskellLinkInfo = provider( }, ) +# HaskellProfLinkInfo exposes the MergedLinkInfo of a target and all of its +# dependencies built for profiling. This allows top-level targets (e.g. +# `haskell_binary`) to be defined with profiling enabled by default. +HaskellProfLinkInfo = provider( + fields = { + "prof_infos": provider_field(typing.Any, default = None), # MergedLinkInfo + }, +) + def merge_haskell_link_infos(deps: list[HaskellLinkInfo]) -> HaskellLinkInfo: merged = {} prof_merged = {} diff --git a/prelude/haskell/util.bzl b/prelude/haskell/util.bzl index 21cbd7b054098..89545e7affae5 100644 --- a/prelude/haskell/util.bzl +++ b/prelude/haskell/util.bzl @@ -10,9 +10,24 @@ load( "@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", + "HaskellLibraryProvider", +) +load( + "@prelude//haskell:link_info.bzl", + "HaskellLinkInfo", + "HaskellProfLinkInfo", +) load( "@prelude//linking:link_info.bzl", "LinkStyle", + "MergedLinkInfo", +) +load( + "@prelude//linking:shared_libraries.bzl", + "SharedLibraryInfo", ) load("@prelude//utils:platform_flavors_util.bzl", "by_platform") load("@prelude//utils:utils.bzl", "flatten") @@ -49,6 +64,66 @@ def _by_platform(ctx: AnalysisContext, xs: list[(str, list[typing.Any])]) -> lis def attr_deps(ctx: AnalysisContext) -> list[Dependency]: return ctx.attrs.deps + _by_platform(ctx, ctx.attrs.platform_deps) +def attr_deps_haskell_link_infos(ctx: AnalysisContext) -> list[HaskellLinkInfo]: + return filter( + None, + [ + d.get(HaskellLinkInfo) + for d in attr_deps(ctx) + ctx.attrs.template_deps + ], + ) + +# DONT CALL THIS FUNCTION, you want attr_deps_haskell_link_infos instead +def attr_deps_haskell_link_infos_sans_template_deps(ctx: AnalysisContext) -> list[HaskellLinkInfo]: + return filter( + None, + [ + d.get(HaskellLinkInfo) + for d in attr_deps(ctx) + ], + ) + +def attr_deps_haskell_lib_infos( + ctx: AnalysisContext, + link_style: LinkStyle, + enable_profiling: bool) -> list[HaskellLibraryInfo]: + if enable_profiling and link_style == LinkStyle("shared"): + fail("Profiling isn't supported when using dynamic linking") + return [ + x.prof_lib[link_style] if enable_profiling else x.lib[link_style] + for x in filter(None, [ + d.get(HaskellLibraryProvider) + for d in attr_deps(ctx) + ctx.attrs.template_deps + ]) + ] + +def attr_deps_merged_link_infos(ctx: AnalysisContext) -> list[MergedLinkInfo]: + return filter( + None, + [ + d.get(MergedLinkInfo) + for d in attr_deps(ctx) + ], + ) + +def attr_deps_profiling_link_infos(ctx: AnalysisContext) -> list[MergedLinkInfo]: + return filter( + None, + [ + d.get(HaskellProfLinkInfo).prof_infos if d.get(HaskellProfLinkInfo) else d.get(MergedLinkInfo) + for d in attr_deps(ctx) + ], + ) + +def attr_deps_shared_library_infos(ctx: AnalysisContext) -> list[SharedLibraryInfo]: + return filter( + None, + [ + d.get(SharedLibraryInfo) + for d in attr_deps(ctx) + ], + ) + def _link_style_extensions(link_style: LinkStyle) -> (str, str): if link_style == LinkStyle("shared"): return ("dyn_o", "dyn_hi") diff --git a/prelude/http_archive/http_archive.bzl b/prelude/http_archive/http_archive.bzl index 0c16016756b02..599e271d6ed73 100644 --- a/prelude/http_archive/http_archive.bzl +++ b/prelude/http_archive/http_archive.bzl @@ -66,8 +66,9 @@ def _unarchive_cmd( archive, "--stdout", "|", - "tar", + "%WINDIR%\\System32\\tar.exe", "-x", + "-P", "-f", "-", _tar_strip_prefix_flags(strip_prefix), diff --git a/prelude/java/java.bzl b/prelude/java/java.bzl index 9a35285df188c..92137a3522e76 100644 --- a/prelude/java/java.bzl +++ b/prelude/java/java.bzl @@ -80,6 +80,7 @@ extra_attributes = { "abi_generation_mode": attrs.option(attrs.enum(AbiGenerationMode), default = None), "javac": attrs.option(attrs.one_of(attrs.dep(), attrs.source()), default = None), "resources_root": attrs.option(attrs.string(), default = None), + "test_class_names_file": attrs.option(attrs.source(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_exec_os_type": buck.exec_os_type_arg(), diff --git a/prelude/java/java_library.bzl b/prelude/java/java_library.bzl index 45525a411aa53..512b6c503b539 100644 --- a/prelude/java/java_library.bzl +++ b/prelude/java/java_library.bzl @@ -88,8 +88,7 @@ def _process_plugins( # Process Javac Plugins if plugin_params: - plugin = plugin_params.processors[0] - args = plugin_params.args.get(plugin, cmd_args()) + plugin, args = plugin_params.processors[0] # Produces "-Xplugin:PluginName arg1 arg2 arg3", as a single argument plugin_and_args = cmd_args(plugin) diff --git a/prelude/java/java_test.bzl b/prelude/java/java_test.bzl index 1077416f7de63..75a0d2d330c7c 100644 --- a/prelude/java/java_test.bzl +++ b/prelude/java/java_test.bzl @@ -112,19 +112,21 @@ def build_junit_test( if ctx.attrs.test_case_timeout_ms: cmd.extend(["--default_test_timeout", str(ctx.attrs.test_case_timeout_ms)]) - expect(tests_java_library_info.library_output != None, "Built test library has no output, likely due to missing srcs") - - class_names = ctx.actions.declare_output("class_names") - list_class_names_cmd = cmd_args([ - java_test_toolchain.list_class_names[RunInfo], - "--jar", - tests_java_library_info.library_output.full_library, - "--sources", - ctx.actions.write("sources.txt", ctx.attrs.srcs), - "--output", - class_names.as_output(), - ]).hidden(ctx.attrs.srcs) - ctx.actions.run(list_class_names_cmd, category = "list_class_names") + if ctx.attrs.test_class_names_file: + class_names = ctx.attrs.test_class_names_file + else: + expect(tests_java_library_info.library_output != None, "Built test library has no output, likely due to missing srcs") + class_names = ctx.actions.declare_output("class_names") + list_class_names_cmd = cmd_args([ + java_test_toolchain.list_class_names[RunInfo], + "--jar", + tests_java_library_info.library_output.full_library, + "--sources", + ctx.actions.write("sources.txt", ctx.attrs.srcs), + "--output", + class_names.as_output(), + ]).hidden(ctx.attrs.srcs) + ctx.actions.run(list_class_names_cmd, category = "list_class_names") cmd.extend(["--test-class-names-file", class_names]) @@ -139,7 +141,7 @@ def build_junit_test( if tests_class_to_source_info != None: transitive_class_to_src_map = merge_class_to_source_map_from_jar( actions = ctx.actions, - name = ctx.attrs.name + ".transitive_class_to_src.json", + name = ctx.label.name + ".transitive_class_to_src.json", java_test_toolchain = java_test_toolchain, relative_to = ctx.label.cell_root if run_from_cell_root else None, deps = [tests_class_to_source_info], diff --git a/prelude/java/java_toolchain.bzl b/prelude/java/java_toolchain.bzl index dbfd1ccc14770..51bbe0637849a 100644 --- a/prelude/java/java_toolchain.bzl +++ b/prelude/java/java_toolchain.bzl @@ -33,10 +33,12 @@ JavaToolchainInfo = provider( "fat_jar_main_class_lib": provider_field(typing.Any, default = None), "gen_class_to_source_map": provider_field(typing.Any, default = None), "gen_class_to_source_map_debuginfo": provider_field(typing.Any, default = None), # optional + "graalvm_java": provider_field(typing.Any, default = None), "is_bootstrap_toolchain": provider_field(typing.Any, default = None), "jar": provider_field(typing.Any, default = None), "jar_builder": provider_field(typing.Any, default = None), "java": provider_field(typing.Any, default = None), + "java_error_handler": provider_field(typing.Any, default = None), "java_for_tests": provider_field(typing.Any, default = None), "javac": provider_field(typing.Any, default = None), "javac_protocol": provider_field(typing.Any, default = None), @@ -53,6 +55,7 @@ JavaToolchainInfo = provider( "src_root_elements": provider_field(typing.Any, default = None), "src_root_prefixes": provider_field(typing.Any, default = None), "target_level": provider_field(typing.Any, default = None), + "use_graalvm_java_for_javacd": provider_field(typing.Any, default = None), "zip_scrubber": provider_field(typing.Any, default = None), }, ) diff --git a/prelude/java/javacd_jar_creator.bzl b/prelude/java/javacd_jar_creator.bzl index dd5efb7c1e6fb..1806facc6ae9e 100644 --- a/prelude/java/javacd_jar_creator.bzl +++ b/prelude/java/javacd_jar_creator.bzl @@ -209,7 +209,7 @@ def create_jar_artifact_javacd( compiler = java_toolchain.javac[DefaultInfo].default_outputs[0] exe, local_only = prepare_cd_exe( qualified_name, - java = java_toolchain.java[RunInfo], + java = java_toolchain.graalvm_java[RunInfo] if java_toolchain.use_graalvm_java_for_javacd else java_toolchain.java[RunInfo], class_loader_bootstrapper = java_toolchain.class_loader_bootstrapper, compiler = compiler, main_class = java_toolchain.javacd_main_class, @@ -288,6 +288,7 @@ def create_jar_artifact_javacd( local_only = local_only, low_pass_filter = False, weight = 2, + error_handler = java_toolchain.java_error_handler, ) library_classpath_jars_tag = actions.artifact_tag() diff --git a/prelude/java/plugins/java_plugin.bzl b/prelude/java/plugins/java_plugin.bzl index ac4129903204f..2636e137a23c6 100644 --- a/prelude/java/plugins/java_plugin.bzl +++ b/prelude/java/plugins/java_plugin.bzl @@ -14,8 +14,7 @@ load( ) PluginParams = record( - processors = field(list[str]), - args = field(dict[str, cmd_args]), + processors = field(list[(str, cmd_args)]), deps = field([JavaPackagingDepTSet, None]), ) @@ -23,22 +22,32 @@ def create_plugin_params(ctx: AnalysisContext, plugins: list[Dependency]) -> [Pl processors = [] plugin_deps = [] + # _wip_java_plugin_arguments keys are providers_label, map to + # target_label to allow lookup with plugin.label.raw_target() + plugin_arguments = { + label.raw_target(): arguments + for label, arguments in ctx.attrs._wip_java_plugin_arguments.items() + } + # Compiler plugin derived from `plugins` attribute - for plugin in filter(None, [x.get(JavaProcessorsInfo) for x in plugins]): - if plugin.type == JavaProcessorsType("plugin"): - if len(plugin.processors) > 1: - fail("Only 1 java compiler plugin is expected. But received: {}".format(plugin.processors)) - processors.append(plugin.processors[0]) - if plugin.deps: - plugin_deps.append(plugin.deps) + for plugin in plugins: + processors_info = plugin.get(JavaProcessorsInfo) + if processors_info != None and processors_info.type == JavaProcessorsType("plugin"): + if len(processors_info.processors) > 1: + fail("Only 1 java compiler plugin is expected. But received: {}".format(processors_info.processors)) + processor = processors_info.processors[0] + if processors_info.deps: + plugin_deps.append(processors_info.deps) + + arguments = plugin_arguments.get(plugin.label.raw_target()) + processors.append((processor, cmd_args(arguments) if arguments != None else cmd_args())) if not processors: return None return PluginParams( - processors = dedupe(processors), + processors = processors, deps = ctx.actions.tset(JavaPackagingDepTSet, children = plugin_deps) if plugin_deps else None, - args = {}, ) def java_plugin_impl(ctx: AnalysisContext) -> list[Provider]: diff --git a/prelude/jvm/cd_jar_creator_util.bzl b/prelude/jvm/cd_jar_creator_util.bzl index 18966fb32a351..e89fe14359f1d 100644 --- a/prelude/jvm/cd_jar_creator_util.bzl +++ b/prelude/jvm/cd_jar_creator_util.bzl @@ -211,7 +211,7 @@ def _get_source_only_abi_compiling_deps(compiling_deps_tset: [JavaCompilingDepsT for d in source_only_abi_deps: info = d.get(JavaLibraryInfo) if not info: - fail("source_only_abi_deps must produce a JavaLibraryInfo but {} does not, please remove it".format(d)) + fail("source_only_abi_deps must produce a JavaLibraryInfo but '{}' does not, please remove it".format(d.label)) if info.library_output: source_only_abi_deps_filter[info.library_output.abi] = True @@ -250,22 +250,28 @@ def encode_ap_params(annotation_processor_properties: AnnotationProcessorPropert return encoded_ap_params def encode_plugin_params(plugin_params: [PluginParams, None]) -> [struct, None]: - # TODO(cjhopman): We should change plugins to not be merged together just like APs. encoded_plugin_params = None if plugin_params: encoded_plugin_params = struct( parameters = [], - pluginProperties = [struct( - canReuseClassLoader = False, - doesNotAffectAbi = False, - supportsAbiGenerationFromSource = False, - processorNames = plugin_params.processors, - classpath = plugin_params.deps.project_as_json("javacd_json") if plugin_params.deps else [], - pathParams = {}, - )], + pluginProperties = [ + encode_plugin_properties(processor, arguments, plugin_params) + for processor, arguments in plugin_params.processors + ], ) return encoded_plugin_params +def encode_plugin_properties(processor: str, arguments: cmd_args, plugin_params: PluginParams) -> struct: + return struct( + canReuseClassLoader = False, + doesNotAffectAbi = False, + supportsAbiGenerationFromSource = False, + processorNames = [processor], + classpath = plugin_params.deps.project_as_json("javacd_json") if plugin_params.deps else [], + pathParams = {}, + arguments = arguments, + ) + def encode_base_jar_command( javac_tool: [str, RunInfo, Artifact, None], target_type: TargetType, diff --git a/prelude/kotlin/kotlin.bzl b/prelude/kotlin/kotlin.bzl index 4c9dd097e2b23..6f60384e95e30 100644 --- a/prelude/kotlin/kotlin.bzl +++ b/prelude/kotlin/kotlin.bzl @@ -35,6 +35,7 @@ extra_attributes = { "abi_generation_mode": attrs.option(attrs.enum(AbiGenerationMode), default = None), "javac": attrs.option(attrs.one_of(attrs.dep(), attrs.source()), default = None), "resources_root": attrs.option(attrs.string(), default = None), + "test_class_names_file": attrs.option(attrs.source(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_exec_os_type": buck.exec_os_type_arg(), diff --git a/prelude/kotlin/kotlin_toolchain.bzl b/prelude/kotlin/kotlin_toolchain.bzl index 248ea5a197cdc..0c863de11aa95 100644 --- a/prelude/kotlin/kotlin_toolchain.bzl +++ b/prelude/kotlin/kotlin_toolchain.bzl @@ -22,6 +22,7 @@ KotlinToolchainInfo = provider( "kosabi_jvm_abi_gen_plugin": provider_field(typing.Any, default = None), "kosabi_stubs_gen_plugin": provider_field(typing.Any, default = None), "kosabi_supported_ksp_providers": provider_field(typing.Any, default = None), + "kotlin_error_handler": provider_field(typing.Any, default = None), "kotlin_home_libraries": provider_field(typing.Any, default = None), "kotlin_stdlib": provider_field(typing.Any, default = None), "kotlinc": provider_field(typing.Any, default = None), diff --git a/prelude/kotlin/kotlincd_jar_creator.bzl b/prelude/kotlin/kotlincd_jar_creator.bzl index 14558b04f666b..24bfa153110b5 100644 --- a/prelude/kotlin/kotlincd_jar_creator.bzl +++ b/prelude/kotlin/kotlincd_jar_creator.bzl @@ -251,7 +251,7 @@ def create_jar_artifact_kotlincd( compiler = kotlin_toolchain.kotlinc[DefaultInfo].default_outputs[0] exe, local_only = prepare_cd_exe( qualified_name, - java = java_toolchain.java[RunInfo], + java = java_toolchain.graalvm_java[RunInfo] if java_toolchain.use_graalvm_java_for_javacd else java_toolchain.java[RunInfo], class_loader_bootstrapper = kotlin_toolchain.class_loader_bootstrapper, compiler = compiler, main_class = kotlin_toolchain.kotlincd_main_class, @@ -331,6 +331,7 @@ def create_jar_artifact_kotlincd( local_only = local_only, low_pass_filter = False, weight = 2, + error_handler = kotlin_toolchain.kotlin_error_handler, ) library_classpath_jars_tag = actions.artifact_tag() diff --git a/prelude/linking/execution_preference.bzl b/prelude/linking/execution_preference.bzl index 92d45adee6110..041ceb7dc0bff 100644 --- a/prelude/linking/execution_preference.bzl +++ b/prelude/linking/execution_preference.bzl @@ -36,13 +36,14 @@ _ActionExecutionAttributes = record( def link_execution_preference_attr(): # The attribute is optional, allowing for None to represent that no preference has been set and we should fallback on the toolchain. return attrs.option(attrs.one_of(attrs.enum(LinkExecutionPreferenceTypes), attrs.dep(providers = [LinkExecutionPreferenceDeterminatorInfo])), default = None, doc = """ - The execution preference for linking. Options are:\n - - any : No preference is set, and the link action will be performed based on buck2's executor configuration.\n - - full_hybrid : The link action will execute both locally and remotely, regardless of buck2's executor configuration (if\n - the executor is capable of hybrid execution). The use_limited_hybrid setting of the hybrid executor is ignored.\n - - local : The link action will execute locally if compatible on current host platform.\n - - local_only : The link action will execute locally, and error if the current platform is not compatible.\n - - remote : The link action will execute remotely if a compatible remote platform exists, otherwise locally.\n + The execution preference for linking. Options are: + + - any : No preference is set, and the link action will be performed based on buck2's executor configuration. + - full_hybrid : The link action will execute both locally and remotely, regardless of buck2's executor configuration (if + the executor is capable of hybrid execution). The use_limited_hybrid setting of the hybrid executor is ignored. + - local : The link action will execute locally if compatible on current host platform. + - local_only : The link action will execute locally, and error if the current platform is not compatible. + - remote : The link action will execute remotely if a compatible remote platform exists, otherwise locally. The default is None, expressing that no preference has been set on the target itself. """) diff --git a/prelude/linking/link_groups.bzl b/prelude/linking/link_groups.bzl index 19582078e0409..6d6bdbef3c0a3 100644 --- a/prelude/linking/link_groups.bzl +++ b/prelude/linking/link_groups.bzl @@ -50,7 +50,8 @@ def merge_link_group_lib_info( name: [str, None] = None, shared_libs: [dict[str, LinkedObject], None] = None, shared_link_infos: [LinkInfos, None] = None, - deps: list[Dependency] = []) -> LinkGroupLibInfo: + deps: list[Dependency] = [], + children: list[LinkGroupLibInfo] = []) -> LinkGroupLibInfo: """ Merge and return link group info libs from deps and the current rule wrapped in a provider. @@ -66,5 +67,6 @@ def merge_link_group_lib_info( libs = gather_link_group_libs( libs = [libs], deps = deps, + children = children, ), ) diff --git a/prelude/linking/link_info.bzl b/prelude/linking/link_info.bzl index cb938a17a9c7e..d9b8cc1b2909b 100644 --- a/prelude/linking/link_info.bzl +++ b/prelude/linking/link_info.bzl @@ -177,6 +177,10 @@ LinkOrdering = enum( "topological", ) +CxxSanitizerRuntimeInfo = provider(fields = { + "runtime_dir": provider_field(Artifact), +}) + def set_link_info_link_whole(info: LinkInfo) -> LinkInfo: linkables = [set_linkable_link_whole(linkable) for linkable in info.linkables] return LinkInfo( diff --git a/prelude/linking/linkable_graph.bzl b/prelude/linking/linkable_graph.bzl index b2feb9ceb43fc..1f73160ed4af9 100644 --- a/prelude/linking/linkable_graph.bzl +++ b/prelude/linking/linkable_graph.bzl @@ -87,6 +87,8 @@ LinkableNode = record( # Whether the node should appear in the android mergemap (which provides information about the original # soname->final merged lib mapping) include_in_android_mergemap = field(bool), + # Don't follow dependents on this node even if has preferred linkage static + ignore_force_static_follows_dependents = field(bool), # Only allow constructing within this file. _private = _DisallowConstruction, @@ -138,13 +140,14 @@ def create_linkable_node( default_soname: str | None, preferred_linkage: Linkage = Linkage("any"), default_link_strategy: LinkStrategy = LinkStrategy("shared"), - deps: list[Dependency] = [], - exported_deps: list[Dependency] = [], + deps: list[Dependency | LinkableGraph] = [], + exported_deps: list[Dependency | LinkableGraph] = [], link_infos: dict[LibOutputStyle, LinkInfos] = {}, shared_libs: dict[str, LinkedObject] = {}, can_be_asset: bool = True, include_in_android_mergemap: bool = True, - linker_flags: [LinkerFlags, None] = None) -> LinkableNode: + linker_flags: [LinkerFlags, None] = None, + ignore_force_static_follows_dependents: bool = False) -> LinkableNode: for output_style in _get_required_outputs_for_linkage(preferred_linkage): expect( output_style in link_infos, @@ -164,6 +167,7 @@ def create_linkable_node( include_in_android_mergemap = include_in_android_mergemap, default_soname = default_soname, linker_flags = linker_flags, + ignore_force_static_follows_dependents = ignore_force_static_follows_dependents, _private = _DisallowConstruction(), ) @@ -171,9 +175,12 @@ def create_linkable_graph_node( ctx: AnalysisContext, linkable_node: [LinkableNode, None] = None, roots: dict[Label, LinkableRootInfo] = {}, - excluded: dict[Label, None] = {}) -> LinkableGraphNode: + excluded: dict[Label, None] = {}, + label: Label | None = None) -> LinkableGraphNode: + if not label: + label = ctx.label return LinkableGraphNode( - label = ctx.label, + label = label, linkable = linkable_node, roots = roots, excluded = excluded, @@ -196,7 +203,7 @@ def create_linkable_graph( deps_labels = {x.label: True for x in graph_deps} if node and node.linkable: - for l in [node.linkable.deps, node.linkable.exported_deps]: + for l in [node.linkable.deps, node.linkable.exported_deps]: # buildifier: disable=confusing-name for d in l: if not d in deps_labels: fail("LinkableNode had {} in its deps, but that label is missing from the node's linkable graph children (`{}`)".format(d, ", ".join(deps_labels))) @@ -208,8 +215,11 @@ def create_linkable_graph( } if node: kwargs["value"] = node + label = node.label + else: + label = ctx.label return LinkableGraph( - label = ctx.label, + label = label, nodes = ctx.actions.tset(LinkableGraphTSet, **kwargs), ) @@ -224,13 +234,16 @@ def get_linkable_graph_node_map_func(graph: LinkableGraph): return get_linkable_graph_node_map -def linkable_deps(deps: list[Dependency]) -> list[Label]: +def linkable_deps(deps: list[Dependency | LinkableGraph]) -> list[Label]: labels = [] for dep in deps: - dep_info = linkable_graph(dep) - if dep_info != None: - labels.append(dep_info.label) + if eval_type(LinkableGraph.type).matches(dep): + labels.append(dep.label) + else: + dep_info = linkable_graph(dep) + if dep_info != None: + labels.append(dep_info.label) return labels diff --git a/prelude/native.bzl b/prelude/native.bzl index 9c37d87c2d419..1bbadd2abc1cf 100644 --- a/prelude/native.bzl +++ b/prelude/native.bzl @@ -203,9 +203,17 @@ def _android_binary_macro_stub( def _android_instrumentation_apk_macro_stub( cpu_filters = None, + primary_dex_patterns = [], **kwargs): + primary_dex_patterns = primary_dex_patterns + [ + "/R^", + "/R$", + # Pin this to the primary for apps with no primary dex classes. + "^com/facebook/buck_generated/AppWithoutResourcesStub^", + ] __rules__["android_instrumentation_apk"]( cpu_filters = _get_valid_cpu_filters(cpu_filters), + primary_dex_patterns = primary_dex_patterns, **kwargs ) diff --git a/prelude/ocaml/ocaml.bzl b/prelude/ocaml/ocaml.bzl index abd652b83bcff..7afc804b04a77 100644 --- a/prelude/ocaml/ocaml.bzl +++ b/prelude/ocaml/ocaml.bzl @@ -689,7 +689,7 @@ def ocaml_library_impl(ctx: AnalysisContext) -> list[Provider]: info_ide = [ DefaultInfo( - default_output = cmxa, + default_output = cmts_nat[0] if cmts_nat else None, other_outputs = [cmd_args(other_outputs_info.info.project_as_args("ide"))], ), ] @@ -784,7 +784,7 @@ def ocaml_binary_impl(ctx: AnalysisContext) -> list[Provider]: info_ide = [ DefaultInfo( - default_output = binary_nat, + default_output = cmts_nat[0] if cmts_nat else None, other_outputs = [cmd_args(other_outputs_info.info.project_as_args("ide"))], ), ] @@ -876,7 +876,7 @@ def ocaml_object_impl(ctx: AnalysisContext) -> list[Provider]: info_ide = [ DefaultInfo( - default_output = obj, + default_output = cmts[0] if cmts else None, other_outputs = [cmd_args(other_outputs_info.info.project_as_args("ide"))], ), ] @@ -958,7 +958,7 @@ def ocaml_shared_impl(ctx: AnalysisContext) -> list[Provider]: info_ide = [ DefaultInfo( - default_output = binary_nat, + default_output = cmts_nat[0] if cmts_nat else None, other_outputs = [cmd_args(other_outputs_info.info.project_as_args("ide"))], ), ] diff --git a/prelude/oss/CHANGELOG.md b/prelude/oss/CHANGELOG.md index 0ab36850df0e3..524da63093fc7 100644 --- a/prelude/oss/CHANGELOG.md +++ b/prelude/oss/CHANGELOG.md @@ -1,3 +1,3 @@ # Buck2 Prelude -* Initial version. +- Initial version. diff --git a/prelude/oss/CONTRIBUTING.md b/prelude/oss/CONTRIBUTING.md index e2f05f03e678f..7f7c52bbb8d53 100644 --- a/prelude/oss/CONTRIBUTING.md +++ b/prelude/oss/CONTRIBUTING.md @@ -1,15 +1,16 @@ # Contributing to Buck2 Prelude -This repository is a subset of . -You can contribute to either that repo, or this repo - changes will be mirrored to both. +This repository is a subset of . You can +contribute to either that repo, or this repo - changes will be mirrored to both. -We want to make contributing to this project as easy and transparent as possible. +We want to make contributing to this project as easy and transparent as +possible. ## Our Development Process -Buck2 Prelude is currently developed in Facebook's internal repositories and then exported -out to GitHub by a Facebook team member; however, we invite you to submit pull -requests as described below. +Buck2 Prelude is currently developed in Facebook's internal repositories and +then exported out to GitHub by a Facebook team member; however, we invite you to +submit pull requests as described below. ## Pull Requests @@ -45,5 +46,6 @@ We use several Python formatters. ## License By contributing to Buck2 Prelude, you agree that your contributions will be -licensed under both the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) -files in the root directory of this source tree. +licensed under both the [LICENSE-MIT](LICENSE-MIT) and +[LICENSE-APACHE](LICENSE-APACHE) files in the root directory of this source +tree. diff --git a/prelude/oss/README.md b/prelude/oss/README.md index e41bdc072c1cf..6830efe26d3ae 100644 --- a/prelude/oss/README.md +++ b/prelude/oss/README.md @@ -1,12 +1,16 @@ # Buck2 Prelude -This repo contains a copy of the Buck2 Prelude, which is often included as a submodule with a Buck2 project. -To obtain a copy of this repo, and set up other details of a Buck2, you should usually run `buck2 init --git`. -Most information can be found on the main [Buck2 GitHub project](https://github.com/facebook/buck2). +This repo contains a copy of the Buck2 Prelude, which is often included as a +submodule with a Buck2 project. To obtain a copy of this repo, and set up other +details of a Buck2, you should usually run `buck2 init --git`. Most information +can be found on the main +[Buck2 GitHub project](https://github.com/facebook/buck2). -Pull requests and issues should be raised at [facebook/buck2](https://github.com/facebook/buck2) as that project -is more closely monitored and contains CI checks. +Pull requests and issues should be raised at +[facebook/buck2](https://github.com/facebook/buck2) as that project is more +closely monitored and contains CI checks. ## License -Buck2 Prelude is both MIT and Apache License, Version 2.0 licensed, as found in the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. +Buck2 Prelude is both MIT and Apache License, Version 2.0 licensed, as found in +the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. diff --git a/prelude/oss/pull_request_template.md b/prelude/oss/pull_request_template.md index 1554e0ee172bb..ab8b597978799 100644 --- a/prelude/oss/pull_request_template.md +++ b/prelude/oss/pull_request_template.md @@ -1,3 +1,7 @@ -IMPORTANT: Please don't raise pull requests here, but at [facebook/buck2](https://github.com/facebook/buck2/pulls). +IMPORTANT: Please don't raise pull requests here, but at +[facebook/buck2](https://github.com/facebook/buck2/pulls). -The [`prelude`](https://github.com/facebook/buck2/tree/main/prelude) directory is a mirror of this repo, but that repo also features CI tests and is more actively monitored. Any PR's landing there will automatically show up here at the same time. +The [`prelude`](https://github.com/facebook/buck2/tree/main/prelude) directory +is a mirror of this repo, but that repo also features CI tests and is more +actively monitored. Any PR's landing there will automatically show up here at +the same time. diff --git a/prelude/prelude.bzl b/prelude/prelude.bzl index 6ef06c1ea3796..ac15950e85302 100644 --- a/prelude/prelude.bzl +++ b/prelude/prelude.bzl @@ -10,5 +10,3 @@ load("@prelude//:native.bzl", _native = "native") # Public symbols in this file become globals everywhere except `bzl` files in prelude. # Additionally, members of `native` struct also become globals in `BUCK` files. native = _native - -# This is a test to get CI to notice me diff --git a/prelude/python/make_py_package.bzl b/prelude/python/make_py_package.bzl index 96faa653d8a7f..b19c59f7aa858 100644 --- a/prelude/python/make_py_package.bzl +++ b/prelude/python/make_py_package.bzl @@ -145,10 +145,12 @@ def make_py_package( srcs.append(pex_modules.extensions.manifest) preload_libraries = _preload_libraries_args(ctx, shared_libraries) + startup_function = generate_startup_function_loader(ctx) manifest_module = generate_manifest_module(ctx, python_toolchain, srcs) common_modules_args, dep_artifacts, debug_artifacts = _pex_modules_common_args( ctx, pex_modules, + [startup_function] if startup_function else [], {name: lib for name, (lib, _) in shared_libraries.items()}, ) @@ -190,6 +192,10 @@ def make_py_package( allow_cache_upload = allow_cache_upload, ) default.sub_targets[style] = make_py_package_providers(pex_providers) + + # cpp binaries already emit a `debuginfo` subtarget with a different format, + # so we opt to use a more specific subtarget + default.sub_targets["par-debuginfo"] = _debuginfo_subtarget(ctx, debug_artifacts) return default def _make_py_package_impl( @@ -325,6 +331,10 @@ def _make_py_package_impl( run_cmd = cmd_args(run_args).hidden([a for a, _ in runtime_files] + hidden_resources), ) +def _debuginfo_subtarget(ctx: AnalysisContext, debug_artifacts: list[(ArgLike, str)]) -> list[Provider]: + out = ctx.actions.write_json("debuginfo.manifest.json", debug_artifacts) + return [DefaultInfo(default_output = out, other_outputs = [a for a, _ in debug_artifacts])] + def _preload_libraries_args(ctx: AnalysisContext, shared_libraries: dict[str, (LinkedObject, bool)]) -> cmd_args: preload_libraries_path = ctx.actions.write( "__preload_libraries.txt", @@ -372,6 +382,7 @@ def _pex_bootstrap_args( def _pex_modules_common_args( ctx: AnalysisContext, pex_modules: PexModules, + extra_manifests: list[ArgLike], shared_libraries: dict[str, LinkedObject]) -> (cmd_args, list[(ArgLike, str)], list[(ArgLike, str)]): srcs = [] src_artifacts = [] @@ -389,6 +400,9 @@ def _pex_modules_common_args( srcs.append(pex_modules.extra_manifests.manifest) src_artifacts.extend(pex_modules.extra_manifests.artifacts) + if extra_manifests: + srcs.extend(extra_manifests) + deps.extend(src_artifacts) resources = pex_modules.manifests.resource_manifests() deps.extend(pex_modules.manifests.resource_artifacts_with_paths()) @@ -544,6 +558,59 @@ def _hidden_resources_error_message(current_target: Label, hidden_resources: lis msg += " {}\n".format(resource) return msg +def generate_startup_function_loader(ctx: AnalysisContext) -> ArgLike: + """ + Generate `__startup_function_loader__.py` used for early bootstrap of a par. + Things that go here are also enumerated in `__manifest__['startup_functions']` + Some examples include: + * static extension finder init + * eager import loader init + * cinderx init + """ + + if ctx.attrs.manifest_module_entries == None: + startup_functions_list = "" + else: + startup_functions_list = "\n".join( + [ + '"' + startup_function + '",' + for _, startup_function in sorted(ctx.attrs.manifest_module_entries["startup_functions"].items()) + ], + ) + + src_startup_functions_path = ctx.actions.write( + "manifest/__startup_function_loader__.py", + """ +import importlib +import warnings + +STARTUP_FUNCTIONS=[{startup_functions_list}] + +def load_startup_functions(): + for func in STARTUP_FUNCTIONS: + mod, sep, func = func.partition(":") + if sep: + try: + module = importlib.import_module(mod) + getattr(module, func)() + except Exception as e: + # TODO: Ignoring errors for now. + warnings.warn( + "Startup function %s (%s:%s) not executed: %s" + % (mod, name, func, e), + stacklevel=1, + ) + + """.format(startup_functions_list = startup_functions_list), + ) + return ctx.actions.write_json( + "manifest/startup_function_loader.manifest", + [ + ["__par__/__startup_function_loader__.py", src_startup_functions_path, "prelude//python:make_py_package.bzl"], + ], + with_inputs = True, + ) + def generate_manifest_module( ctx: AnalysisContext, python_toolchain: PythonToolchainInfo, diff --git a/prelude/python/python_binary.bzl b/prelude/python/python_binary.bzl index 90f70465f68ea..30d62f1d689cd 100644 --- a/prelude/python/python_binary.bzl +++ b/prelude/python/python_binary.bzl @@ -98,6 +98,7 @@ load( ) load(":source_db.bzl", "create_dbg_source_db", "create_python_source_db_info", "create_source_db", "create_source_db_no_deps") load(":toolchain.bzl", "NativeLinkStrategy", "PackageStyle", "PythonPlatformInfo", "PythonToolchainInfo", "get_package_style", "get_platform_attr") +load(":typing.bzl", "create_per_target_type_check") OmnibusMetadataInfo = provider( # @unsorted-dict-items @@ -395,10 +396,28 @@ def python_executable( exe.sub_targets.update({ "dbg-source-db": [dbg_source_db], "library-info": [library_info], + "main": [DefaultInfo(default_output = ctx.actions.write_json("main.json", main))], "source-db": [source_db], "source-db-no-deps": [source_db_no_deps, create_python_source_db_info(library_info.manifests)], }) + # Type check + type_checker = python_toolchain.type_checker + if type_checker != None: + exe.sub_targets.update({ + "typecheck": [ + create_per_target_type_check( + ctx, + type_checker, + src_manifest, + python_deps, + typeshed_stubs = python_toolchain.typeshed_stubs, + py_version = ctx.attrs.py_version_for_type_checking, + typing_enabled = ctx.attrs.typing, + ), + ], + }) + return exe def create_dep_report( @@ -714,6 +733,19 @@ def python_binary_impl(ctx: AnalysisContext) -> list[Provider]: if main_module.endswith(".py"): main_module = main_module[:-3] + if "python-version=3.8" in ctx.attrs.labels: + # buildifier: disable=print + print(( + "\033[1;33m \u26A0 " + + "{0} 3.8 is EOL, and is going away by the end of H1 2024. " + + "Upgrade //{1}:{2} to {0} 3.10 now to avoid breakages. " + + "https://fburl.com/py38-sunsetting \033[0m" + ).format( + "Cinder" if "python-flavor=cinder" in ctx.attrs.labels else "Python", + ctx.label.package, + ctx.attrs.name, + )) + if main_module != None: main = (EntryPointKind("module"), main_module) else: diff --git a/prelude/python/python_library.bzl b/prelude/python/python_library.bzl index e4238f8ae4e8c..295a076ca1a66 100644 --- a/prelude/python/python_library.bzl +++ b/prelude/python/python_library.bzl @@ -53,6 +53,7 @@ load(":needed_coverage.bzl", "PythonNeededCoverageInfo") load(":python.bzl", "PythonLibraryInfo", "PythonLibraryManifests", "PythonLibraryManifestsTSet") load(":source_db.bzl", "create_python_source_db_info", "create_source_db", "create_source_db_no_deps") load(":toolchain.bzl", "PythonToolchainInfo") +load(":typing.bzl", "create_per_target_type_check") def dest_prefix(label: Label, base_module: [None, str]) -> str: """ @@ -310,6 +311,22 @@ def python_library_impl(ctx: AnalysisContext) -> list[Provider]: # Source DBs. sub_targets["source-db"] = [create_source_db(ctx, src_type_manifest, deps)] sub_targets["source-db-no-deps"] = [create_source_db_no_deps(ctx, src_types), create_python_source_db_info(library_info.manifests)] + + # Type check + type_checker = python_toolchain.type_checker + if type_checker != None: + sub_targets["typecheck"] = [ + create_per_target_type_check( + ctx, + type_checker, + src_type_manifest, + deps, + typeshed_stubs = python_toolchain.typeshed_stubs, + py_version = ctx.attrs.py_version_for_type_checking, + typing_enabled = ctx.attrs.typing, + ), + ] + providers.append(DefaultInfo(sub_targets = sub_targets)) # Create, augment and provide the linkable graph. diff --git a/prelude/python/sourcedb/filter.bxl b/prelude/python/sourcedb/filter.bxl new file mode 100644 index 0000000000000..9cbbbe214c5d7 --- /dev/null +++ b/prelude/python/sourcedb/filter.bxl @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +BUCK_PYTHON_RULE_KINDS = [ + "python_binary", + "python_library", + "python_test", +] +BUCK_PYTHON_RULE_KIND_QUERY = "|".join(BUCK_PYTHON_RULE_KINDS) + +def filter_root_targets( + query: bxl.CqueryContext, + target_patterns: typing.Any) -> bxl.ConfiguredTargetSet: + # Find all Pure-Python targets + candidate_targets = ctarget_set() + for pattern in target_patterns: + candidate_targets += query.kind( + BUCK_PYTHON_RULE_KIND_QUERY, + pattern, + ) + + # Don't check generated rules + filtered_targets = candidate_targets - query.attrfilter( + "labels", + "generated", + candidate_targets, + ) + + # Provide an opt-out label + filtered_targets = filtered_targets - query.attrfilter( + "labels", + "no_pyre", + candidate_targets, + ) + return filtered_targets + +def do_filter( + query: bxl.CqueryContext, + target_patterns: typing.Any) -> list[ConfiguredTargetLabel]: + root_targets = filter_root_targets(query, target_patterns) + return [root_target.label for root_target in root_targets] + +def _do_filter_entry_point(ctx: bxl.Context) -> None: + query = ctx.cquery() + targets = do_filter( + query, + [query.eval(target) for target in ctx.cli_args.target], + ) + ctx.output.print_json([target.raw_target() for target in targets]) + +filter = bxl_main( + doc = ( + "Expand target patterns and look for all targets in immediate sources " + + "that will be built by Pyre." + ), + impl = _do_filter_entry_point, + cli_args = { + "target": cli_args.list(cli_args.string()), + }, +) diff --git a/prelude/python/sourcedb/query.bxl b/prelude/python/sourcedb/query.bxl index 8152d15dbdd9b..3b79a3b691d08 100644 --- a/prelude/python/sourcedb/query.bxl +++ b/prelude/python/sourcedb/query.bxl @@ -7,39 +7,7 @@ load("@prelude//python:python.bzl", "PythonLibraryManifestsTSet") load("@prelude//python:source_db.bzl", "PythonSourceDBInfo") - -BUCK_PYTHON_RULE_KINDS = [ - "python_binary", - "python_library", - "python_test", -] -BUCK_PYTHON_RULE_KIND_QUERY = "|".join(BUCK_PYTHON_RULE_KINDS) - -def _filter_root_targets( - query: bxl.CqueryContext, - target_patterns: typing.Any) -> bxl.ConfiguredTargetSet: - # Find all Pure-Python targets - candidate_targets = ctarget_set() - for pattern in target_patterns: - candidate_targets += query.kind( - BUCK_PYTHON_RULE_KIND_QUERY, - pattern, - ) - - # Don't check generated rules - filtered_targets = candidate_targets - query.attrfilter( - "labels", - "generated", - candidate_targets, - ) - - # Provide an opt-out label - filtered_targets = filtered_targets - query.attrfilter( - "labels", - "no_pyre", - candidate_targets, - ) - return filtered_targets +load("@prelude//python/sourcedb/filter.bxl", "filter_root_targets") def _get_python_library_manifests_from_analysis_result( analysis_result: bxl.AnalysisResult) -> [PythonLibraryManifestsTSet, None]: @@ -73,7 +41,7 @@ def get_python_library_manifests_tset_from_target_patterns( query: bxl.CqueryContext, actions: AnalysisActions, target_patterns: typing.Any) -> PythonLibraryManifestsTSet: - root_targets = _filter_root_targets(query, target_patterns) + root_targets = filter_root_targets(query, target_patterns) return get_python_library_manifests_tset_from_targets(ctx, actions, root_targets) def do_query( @@ -81,11 +49,13 @@ def do_query( query: bxl.CqueryContext, actions: AnalysisActions, target_patterns: typing.Any) -> list[ConfiguredTargetLabel]: - manifests_of_transitive_dependencies = get_python_library_manifests_tset_from_target_patterns( - ctx, - query, - actions, - target_patterns, + manifests_of_transitive_dependencies = ( + get_python_library_manifests_tset_from_target_patterns( + ctx, + query, + actions, + target_patterns, + ) ) return [ manifest.label.configured_target() @@ -96,7 +66,12 @@ def do_query( def _do_query_entry_point(ctx: bxl.Context) -> None: query = ctx.cquery() actions = ctx.bxl_actions().actions - targets = do_query(ctx, query, actions, [query.eval(target) for target in ctx.cli_args.target]) + targets = do_query( + ctx, + query, + actions, + [query.eval(target) for target in ctx.cli_args.target], + ) ctx.output.print_json([target.raw_target() for target in targets]) query = bxl_main( diff --git a/prelude/python/toolchain.bzl b/prelude/python/toolchain.bzl index 075c8d835aa01..62bd8be3629d2 100644 --- a/prelude/python/toolchain.bzl +++ b/prelude/python/toolchain.bzl @@ -68,6 +68,8 @@ PythonToolchainInfo = provider( "make_py_package_modules": provider_field(typing.Any, default = None), "pex_executor": provider_field(typing.Any, default = None), "pex_extension": provider_field(typing.Any, default = None), + "type_checker": provider_field(typing.Any, default = None), + "typeshed_stubs": provider_field(typing.Any, default = []), "emit_omnibus_metadata": provider_field(typing.Any, default = None), "fail_with_message": provider_field(typing.Any, default = None), "emit_dependency_metadata": provider_field(typing.Any, default = None), diff --git a/prelude/python/tools/__test_main__.py b/prelude/python/tools/__test_main__.py index 1ce6a946f8de6..d699a7e9cc58b 100644 --- a/prelude/python/tools/__test_main__.py +++ b/prelude/python/tools/__test_main__.py @@ -33,12 +33,9 @@ import traceback import unittest import warnings +from importlib.machinery import PathFinder -with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - import imp - try: from StringIO import StringIO # type: ignore except ImportError: @@ -88,7 +85,7 @@ def include(self, path): return not self.omit(path) -class DebugWipeFinder: +class DebugWipeFinder(PathFinder): """ PEP 302 finder that uses a DebugWipeLoader for all files which do not need coverage @@ -97,28 +94,15 @@ class DebugWipeFinder: def __init__(self, matcher): self.matcher = matcher - def find_module(self, fullname, path=None): - _, _, basename = fullname.rpartition(".") - try: - fd, pypath, (_, _, kind) = imp.find_module(basename, path) - except Exception: - # Finding without hooks using the imp module failed. One reason - # could be that there is a zip file on sys.path. The imp module - # does not support loading from there. Leave finding this module to - # the others finders in sys.meta_path. + def find_spec(self, fullname, path=None, target=None): + spec = super().find_spec(fullname, path=path, target=target) + if spec is None or spec.origin is None: return None - - if hasattr(fd, "close"): - fd.close() - if kind != imp.PY_SOURCE: + if not spec.origin.endswith(".py"): return None - if self.matcher.include(pypath): + if self.matcher.include(spec.origin): return None - """ - This is defined to match CPython's PyVarObject struct - """ - class PyVarObject(ctypes.Structure): _fields_ = [ ("ob_refcnt", ctypes.c_long), @@ -132,7 +116,7 @@ class DebugWipeLoader(SourceFileLoader): """ def get_code(self, fullname): - code = super(DebugWipeLoader, self).get_code(fullname) + code = super().get_code(fullname) if code: # Ideally we'd do # code.co_lnotab = b'' @@ -142,7 +126,9 @@ def get_code(self, fullname): code_impl.ob_size = 0 return code - return DebugWipeLoader(fullname, pypath) + if isinstance(spec.loader, SourceFileLoader): + spec.loader = DebugWipeLoader(fullname, spec.origin) + return spec def optimize_for_coverage(cov, include_patterns, omit_patterns): @@ -200,8 +186,7 @@ def fileno(self): return self._fileno -# pyre-fixme[11]: Annotation `unittest._TextTestResult` is not defined as a type. -class BuckTestResult(unittest._TextTestResult): +class BuckTestResult(unittest.TextTestResult): """ Our own TestResult class that outputs data in a format that can be easily parsed by buck's test runner. diff --git a/prelude/python/tools/make_par/sitecustomize.py b/prelude/python/tools/make_par/sitecustomize.py index 5b29b8225e685..2066c46756a1b 100644 --- a/prelude/python/tools/make_par/sitecustomize.py +++ b/prelude/python/tools/make_par/sitecustomize.py @@ -6,28 +6,27 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -import importlib +from __future__ import annotations + import multiprocessing.util as mp_util import os import sys import threading +import warnings from importlib.machinery import PathFinder from importlib.util import module_from_spec lock = threading.Lock() -# pyre-fixme[3]: Return type must be annotated. -# pyre-fixme[2]: Parameter must be annotated. -def __patch_spawn(var_names, saved_env): +def __patch_spawn(var_names: tuple[str, ...], saved_env: dict[str, str]) -> None: std_spawn = mp_util.spawnv_passfds # pyre-fixme[53]: Captured variable `std_spawn` is not annotated. # pyre-fixme[53]: Captured variable `saved_env` is not annotated. # pyre-fixme[53]: Captured variable `var_names` is not annotated. - # pyre-fixme[3]: Return type must be annotated. # pyre-fixme[2]: Parameter must be annotated. - def spawnv_passfds(path, args, passfds): + def spawnv_passfds(path, args, passfds) -> None | int: with lock: try: for var in var_names: @@ -44,9 +43,7 @@ def spawnv_passfds(path, args, passfds): mp_util.spawnv_passfds = spawnv_passfds -# pyre-fixme[3]: Return type must be annotated. -# pyre-fixme[2]: Parameter must be annotated. -def __clear_env(patch_spawn=True): +def __clear_env(patch_spawn: bool = True) -> None: saved_env = {} darwin_vars = ("DYLD_LIBRARY_PATH", "DYLD_INSERT_LIBRARIES") linux_vars = ("LD_LIBRARY_PATH", "LD_PRELOAD") @@ -72,26 +69,16 @@ def __clear_env(patch_spawn=True): __patch_spawn(var_names, saved_env) -# pyre-fixme[3]: Return type must be annotated. -def __startup__(): - for name, var in os.environ.items(): - if name.startswith("STARTUP_"): - name, sep, func = var.partition(":") - if sep: - try: - module = importlib.import_module(name) - getattr(module, func)() - except Exception as e: - # TODO: Ignoring errors for now. The way to properly fix this should be to make - # sure we are still at the same binary that configured `STARTUP_` before importing. - print( - "Error running startup function %s:%s: %s" % (name, func, e), - file=sys.stderr, - ) - - -# pyre-fixme[3]: Return type must be annotated. -def __passthrough_exec_module(): +def __startup__() -> None: + try: + from __par__.__startup_function_loader__ import load_startup_functions + + load_startup_functions() + except Exception: + warnings.warn("could not load startup functions", stacklevel=1) + + +def __passthrough_exec_module() -> None: # Delegate this module execution to the next module in the path, if any, # effectively making this sitecustomize.py a passthrough module. spec = PathFinder.find_spec( diff --git a/prelude/python/tools/static_extension_finder.py b/prelude/python/tools/static_extension_finder.py index 9b278d3b7d68e..f3be8f919ac77 100644 --- a/prelude/python/tools/static_extension_finder.py +++ b/prelude/python/tools/static_extension_finder.py @@ -5,8 +5,6 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -import sys -from importlib.machinery import ModuleSpec # Add a try except to force eager importing try: @@ -17,6 +15,9 @@ class StaticExtensionFinder: + # pyre-fixme + ModuleSpec = None + @classmethod # pyre-fixme[3]: Return type must be annotated. # pyre-fixme[2]: Parameter must be annotated. @@ -25,16 +26,22 @@ def find_spec(cls, fullname, path, target=None): Use fullname to look up the PyInit function in the main binary. Returns None if not present. This allows importing CExtensions that have been statically linked in. """ + if not fullname: return None if not _check_module(fullname): return None - spec = ModuleSpec( + spec = cls.ModuleSpec( fullname, StaticExtensionLoader, origin="static-extension", is_package=False ) return spec -# pyre-fixme[3]: Return type must be annotated. -def _initialize(): +def _initialize() -> None: + # This imports are here to avoid tricking circular dependencies. see S389486 + import sys + from importlib.machinery import ModuleSpec + + StaticExtensionFinder.ModuleSpec = ModuleSpec + sys.meta_path.insert(0, StaticExtensionFinder) diff --git a/prelude/python/tools/static_extension_utils.cpp b/prelude/python/tools/static_extension_utils.cpp index f35e2a682068e..1470561cbd721 100644 --- a/prelude/python/tools/static_extension_utils.cpp +++ b/prelude/python/tools/static_extension_utils.cpp @@ -24,15 +24,13 @@ namespace { static PyObject* _create_module(PyObject* self, PyObject* spec) { PyObject* name; PyObject* mod; - const char* oldcontext; name = PyObject_GetAttrString(spec, "name"); if (name == nullptr) { return nullptr; } - // TODO private api usage - mod = _PyImport_FindExtensionObject(name, name); + mod = PyImport_GetModule(name); if (mod || PyErr_Occurred()) { Py_DECREF(name); Py_XINCREF(mod); @@ -58,7 +56,15 @@ static PyObject* _create_module(PyObject* self, PyObject* spec) { PyObject* modules = nullptr; PyModuleDef* def; - oldcontext = _Py_PackageContext; + +#if PY_VERSION_HEX >= 0x030C0000 + // Use our custom Python 3.12 C-API to call the statically linked module init + // function + mod = _Ci_PyImport_CallInitFuncWithContext(namestr.c_str(), initfunc); +#else + // In Python 3.10 (and earlier) we need to handle package context swapping + // ourselves + const char* oldcontext = _Py_PackageContext; _Py_PackageContext = namestr.c_str(); if (_Py_PackageContext == nullptr) { _Py_PackageContext = oldcontext; @@ -67,6 +73,7 @@ static PyObject* _create_module(PyObject* self, PyObject* spec) { } mod = initfunc(); _Py_PackageContext = oldcontext; +#endif if (mod == nullptr) { Py_DECREF(name); return nullptr; diff --git a/prelude/python/typing.bzl b/prelude/python/typing.bzl new file mode 100644 index 0000000000000..a856095b48d0a --- /dev/null +++ b/prelude/python/typing.bzl @@ -0,0 +1,79 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@prelude//:artifacts.bzl", "ArtifactGroupInfo") +load("@prelude//python:python.bzl", "PythonLibraryInfo") +load( + ":manifest.bzl", + "ManifestInfo", # @unused Used as a type + "create_manifest_for_source_map", +) +load(":python.bzl", "PythonLibraryManifestsTSet") + +def create_typeshed_manifest_info( + ctx: AnalysisContext, + typeshed_deps: list[Dependency]) -> ManifestInfo: + # NOTE(grievejia): This assumes that if multiple typeshed targets offer + # the same stub file, the target that comes later wins. + srcs = { + artifact.short_path: artifact + for typeshed_dep in typeshed_deps + for artifact in typeshed_dep[ArtifactGroupInfo].artifacts + } + return create_manifest_for_source_map(ctx, "typeshed", srcs) + +def create_per_target_type_check( + ctx: AnalysisContext, + executable: RunInfo, + srcs: ManifestInfo | None, + deps: list[PythonLibraryInfo], + typeshed_stubs: list[Dependency], + py_version: str | None, + typing_enabled: bool) -> DefaultInfo: + output_file_name = "type_check_result.json" + if not typing_enabled: + # Use empty dict to signal that no type checking was performed. + output_file = ctx.actions.write_json(output_file_name, {}) + else: + cmd = cmd_args(executable) + cmd.add(cmd_args("check")) + + # Source artifacts + source_manifests = [] + if srcs != None: + source_manifests = [srcs.manifest] + cmd.hidden([a for a, _ in srcs.artifacts]) + + # Dep artifacts + dep_manifest_tset = ctx.actions.tset(PythonLibraryManifestsTSet, children = [d.manifests for d in deps]) + dep_manifests = dep_manifest_tset.project_as_args("source_type_manifests") + cmd.hidden(dep_manifest_tset.project_as_args("source_type_artifacts")) + + # Typeshed artifacts + if len(typeshed_stubs) > 0: + typeshed_manifest_info = create_typeshed_manifest_info(ctx, typeshed_stubs) + cmd.hidden([a for a, _ in typeshed_manifest_info.artifacts]) + typeshed_manifest = typeshed_manifest_info.manifest + else: + typeshed_manifest = None + + # Create input configs + input_config = { + "dependencies": dep_manifests, + "py_version": py_version, + "sources": source_manifests, + "typeshed": typeshed_manifest, + } + + input_file = ctx.actions.write_json("type_check_config.json", input_config, with_inputs = True) + output_file = ctx.actions.declare_output(output_file_name) + cmd.add(cmd_args(input_file)) + cmd.add(cmd_args(output_file.as_output(), format = "--output={}")) + + ctx.actions.run(cmd, category = "type_check") + + return DefaultInfo(default_output = output_file) diff --git a/prelude/rules.bzl b/prelude/rules.bzl index 5394ad635effe..9653302b8fe51 100644 --- a/prelude/rules.bzl +++ b/prelude/rules.bzl @@ -5,6 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//:buck2_compatibility.bzl", "check_buck2_compatibility") load("@prelude//configurations:rules.bzl", _config_implemented_rules = "implemented_rules") load("@prelude//decls/common.bzl", "prelude_rule") load("@prelude//is_full_meta_repo.bzl", "is_full_meta_repo") @@ -86,11 +87,18 @@ def _mk_rule(rule_spec: typing.Any, extra_attrs: dict[str, typing.Any] = dict(), extra_args.setdefault("is_configuration_rule", name in _config_implemented_rules) extra_args.setdefault("is_toolchain_rule", name in toolchain_rule_names) return rule( - impl = impl, + impl = buck2_compatibility_check_wrapper(impl), attrs = attributes, **extra_args ) +def buck2_compatibility_check_wrapper(impl) -> typing.Callable: + def buck2_compatibility_shim(ctx: AnalysisContext) -> [list[Provider], Promise]: + check_buck2_compatibility(ctx) + return impl(ctx) + + return buck2_compatibility_shim + def _flatten_decls(): decls = {} for decl_set in rule_decl_records: diff --git a/prelude/rules_impl.bzl b/prelude/rules_impl.bzl index 74fa84c4a8b25..85dc45ea27bc4 100644 --- a/prelude/rules_impl.bzl +++ b/prelude/rules_impl.bzl @@ -24,12 +24,14 @@ load("@prelude//go:coverage.bzl", "GoCoverageMode") load("@prelude//go:go_binary.bzl", "go_binary_impl") load("@prelude//go:go_exported_library.bzl", "go_exported_library_impl") load("@prelude//go:go_library.bzl", "go_library_impl") +load("@prelude//go:go_stdlib.bzl", "go_stdlib_impl") load("@prelude//go:go_test.bzl", "go_test_impl") -load("@prelude//haskell:compile.bzl", "HaskellLibraryProvider") +load("@prelude//go/transitions:defs.bzl", "cgo_enabled_attr", "compile_shared_attr", "go_binary_transition", "go_exported_library_transition", "go_test_transition") load("@prelude//haskell:haskell.bzl", "haskell_binary_impl", "haskell_library_impl", "haskell_prebuilt_library_impl") load("@prelude//haskell:haskell_ghci.bzl", "haskell_ghci_impl") load("@prelude//haskell:haskell_haddock.bzl", "haskell_haddock_impl") load("@prelude//haskell:haskell_ide.bzl", "haskell_ide_impl") +load("@prelude//haskell:library_info.bzl", "HaskellLibraryProvider") load("@prelude//http_archive:http_archive.bzl", "http_archive_impl") load("@prelude//java:java.bzl", _java_extra_attributes = "extra_attributes", _java_implemented_rules = "implemented_rules") load("@prelude//js:js.bzl", _js_extra_attributes = "extra_attributes", _js_implemented_rules = "implemented_rules") @@ -169,6 +171,7 @@ extra_implemented_rules = struct( go_exported_library = go_exported_library_impl, go_library = go_library_impl, go_test = go_test_impl, + go_stdlib = go_stdlib_impl, #haskell haskell_library = haskell_library_impl, @@ -369,8 +372,10 @@ inlined_extra_attributes = { # go "cgo_library": { "embedcfg": attrs.option(attrs.source(allow_directory = False), default = None), + "_compile_shared": compile_shared_attr, "_cxx_toolchain": toolchains_common.cxx(), "_exec_os_type": buck.exec_os_type_arg(), + "_go_stdlib": attrs.default_only(attrs.dep(default = "prelude//go/tools:stdlib")), "_go_toolchain": toolchains_common.go(), }, # csharp @@ -416,15 +421,26 @@ inlined_extra_attributes = { "embedcfg": attrs.option(attrs.source(allow_directory = False), default = None), "resources": attrs.list(attrs.one_of(attrs.dep(), attrs.source(allow_directory = True)), default = []), "_exec_os_type": buck.exec_os_type_arg(), + "_go_stdlib": attrs.default_only(attrs.dep(default = "prelude//go/tools:stdlib")), "_go_toolchain": toolchains_common.go(), }, "go_exported_library": { "embedcfg": attrs.option(attrs.source(allow_directory = False), default = None), "_exec_os_type": buck.exec_os_type_arg(), + "_go_stdlib": attrs.default_only(attrs.dep(default = "prelude//go/tools:stdlib")), "_go_toolchain": toolchains_common.go(), }, "go_library": { "embedcfg": attrs.option(attrs.source(allow_directory = False), default = None), + "_cgo_enabled": cgo_enabled_attr, + "_compile_shared": compile_shared_attr, + "_go_stdlib": attrs.default_only(attrs.dep(default = "prelude//go/tools:stdlib")), + "_go_toolchain": toolchains_common.go(), + }, + "go_stdlib": { + "_cgo_enabled": cgo_enabled_attr, + "_compile_shared": compile_shared_attr, + "_exec_os_type": buck.exec_os_type_arg(), "_go_toolchain": toolchains_common.go(), }, "go_test": { @@ -432,6 +448,7 @@ inlined_extra_attributes = { "embedcfg": attrs.option(attrs.source(allow_directory = False), default = None), "resources": attrs.list(attrs.source(allow_directory = True), default = []), "_exec_os_type": buck.exec_os_type_arg(), + "_go_stdlib": attrs.default_only(attrs.dep(default = "prelude//go/tools:stdlib")), "_go_toolchain": toolchains_common.go(), "_testmaingen": attrs.default_only(attrs.exec_dep(default = "prelude//go/tools:testmaingen")), }, @@ -587,6 +604,9 @@ extra_attributes = struct(**all_extra_attributes) transitions = { "android_binary": constraint_overrides_transition, "apple_resource": apple_resource_transition, + "go_binary": go_binary_transition, + "go_exported_library": go_exported_library_transition, + "go_test": go_test_transition, "python_binary": constraint_overrides_transition, "python_test": constraint_overrides_transition, } diff --git a/prelude/rust/build.bzl b/prelude/rust/build.bzl index b983bfc9ee654..7f4a2a5c03052 100644 --- a/prelude/rust/build.bzl +++ b/prelude/rust/build.bzl @@ -5,7 +5,11 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//:artifact_tset.bzl", "project_artifacts") +load( + "@prelude//:artifact_tset.bzl", + "ArtifactTSet", # @unused Used as a type + "project_artifacts", +) load("@prelude//:local_only.bzl", "link_cxx_binary_locally") load("@prelude//:paths.bzl", "paths") load("@prelude//:resources.bzl", "create_resource_db", "gather_resources") @@ -27,15 +31,15 @@ load( load( "@prelude//linking:link_info.bzl", "LinkArgs", - "LinkStyle", + "LinkStrategy", # @unused Used as a type "get_link_args_for_strategy", - "to_link_strategy", ) load( "@prelude//linking:shared_libraries.bzl", "merge_shared_libraries", "traverse_shared_library_info", ) +load("@prelude//linking:strip.bzl", "strip_debug_info") load("@prelude//os_lookup:defs.bzl", "OsLookup") load("@prelude//utils:cmd_script.bzl", "ScriptOs", "cmd_script") load("@prelude//utils:set.bzl", "set") @@ -68,28 +72,30 @@ load( "RustCxxLinkGroupInfo", #@unused Used as a type "RustDependency", "RustLinkInfo", - "RustLinkStyleInfo", "attr_crate", "attr_simple_crate_for_filenames", "get_available_proc_macros", "inherited_external_debug_info", "inherited_merged_link_infos", + "inherited_rust_external_debug_info", "inherited_shared_libs", "normalize_crate", "resolve_rust_deps", - "style_info", + "strategy_info", ) load(":resources.bzl", "rust_attr_resources") load(":rust_toolchain.bzl", "PanicRuntime", "RustToolchainInfo") RustcOutput = record( output = field(Artifact), + stripped_output = field(Artifact), diag = field(dict[str, Artifact]), pdb = field([Artifact, None]), dwp_output = field([Artifact, None]), # Zero or more Split DWARF debug info files are emitted into this directory # with unpredictable filenames. dwo_output_directory = field([Artifact, None]), + extra_external_debug_info = field(list[ArtifactTSet]), ) def compile_context(ctx: AnalysisContext) -> CompileContext: @@ -137,8 +143,6 @@ def compile_context(ctx: AnalysisContext) -> CompileContext: linker_args = linker, clippy_wrapper = clippy_wrapper, common_args = {}, - flagfiles_for_extern = {}, - flagfiles_for_crate_map = {}, transitive_dependency_dirs = {}, ) @@ -162,7 +166,6 @@ def generate_rustdoc( # rather than full .rlibs emit = Emit("metadata"), params = params, - dep_link_style = params.dep_link_style, default_roots = default_roots, is_rustdoc_test = False, ) @@ -171,11 +174,9 @@ def generate_rustdoc( output = ctx.actions.declare_output(subdir) plain_env, path_env = _process_env(compile_ctx, ctx.attrs.env, exec_is_windows) + plain_env["RUSTDOC_BUCK_TARGET"] = cmd_args(str(ctx.label.raw_target())) rustdoc_cmd = cmd_args( - [cmd_args("--env=", k, "=", v, delimiter = "") for k, v in plain_env.items()], - [cmd_args("--path-env=", k, "=", v, delimiter = "") for k, v in path_env.items()], - cmd_args(str(ctx.label.raw_target()), format = "--env=RUSTDOC_BUCK_TARGET={}"), toolchain_info.rustdoc, toolchain_info.rustdoc_flags, ctx.attrs.rustdoc_flags, @@ -189,6 +190,7 @@ def generate_rustdoc( url_prefix = toolchain_info.extern_html_root_url_prefix if url_prefix != None: # Flag --extern-html-root-url used below is only supported on nightly. + plain_env["RUSTC_BOOTSTRAP"] = cmd_args("1") rustdoc_cmd.add("-Zunstable-options") for dep in resolve_rust_deps(ctx, compile_ctx.dep_ctx): @@ -199,7 +201,8 @@ def generate_rustdoc( if dep.name: name = normalize_crate(dep.name) else: - name = dep.info.crate + # TODO: resolve this using dynamic (if set), see comment on D52476603 + name = dep.info.crate.simple rustdoc_cmd.add( "--extern-html-root-url={}={}/{}:{}" @@ -208,10 +211,16 @@ def generate_rustdoc( rustdoc_cmd.hidden(toolchain_info.rustdoc, compile_ctx.symlinked_srcs) + rustdoc_cmd_action = cmd_args( + [cmd_args("--env=", k, "=", v, delimiter = "") for k, v in plain_env.items()], + [cmd_args("--path-env=", k, "=", v, delimiter = "") for k, v in path_env.items()], + rustdoc_cmd, + ) + rustdoc_cmd = _long_command( ctx = ctx, exe = toolchain_info.rustc_action, - args = rustdoc_cmd, + args = rustdoc_cmd_action, argfile_name = "{}.args".format(subdir), ) @@ -222,10 +231,10 @@ def generate_rustdoc( def generate_rustdoc_test( ctx: AnalysisContext, compile_ctx: CompileContext, - link_style: LinkStyle, - library: RustLinkStyleInfo, + link_strategy: LinkStrategy, + rlib: Artifact, params: BuildParams, - default_roots: list[str]) -> (cmd_args, dict[str, cmd_args]): + default_roots: list[str]) -> cmd_args: exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows" toolchain_info = compile_ctx.toolchain_info @@ -240,7 +249,7 @@ def generate_rustdoc_test( resources = create_resource_db( ctx = ctx, name = "doctest/resources.json", - binary = library.rlib, + binary = rlib, resources = flatten_dict(gather_resources( label = ctx.label, resources = rust_attr_resources(ctx), @@ -250,7 +259,7 @@ def generate_rustdoc_test( # Gather and setup symlink tree of transitive shared library deps. shared_libs = {} - if link_style == LinkStyle("shared"): + if link_strategy == LinkStrategy("shared"): shlib_info = merge_shared_libraries( ctx.actions, deps = inherited_shared_libs(ctx, doc_dep_ctx), @@ -258,7 +267,7 @@ def generate_rustdoc_test( for soname, shared_lib in traverse_shared_library_info(shlib_info).items(): shared_libs[soname] = shared_lib.lib executable_args = executable_shared_lib_arguments( - ctx.actions, + ctx, compile_ctx.cxx_toolchain_info, resources, shared_libs, @@ -270,7 +279,6 @@ def generate_rustdoc_test( dep_ctx = doc_dep_ctx, emit = Emit("link"), params = params, - dep_link_style = params.dep_link_style, default_roots = default_roots, is_rustdoc_test = True, ) @@ -283,8 +291,7 @@ def generate_rustdoc_test( get_link_args_for_strategy( ctx, inherited_merged_link_infos(ctx, doc_dep_ctx), - # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy. - to_link_strategy(link_style), + link_strategy, ), ], "{}-{}".format(common_args.subdir, common_args.tempfile), @@ -303,14 +310,30 @@ def generate_rustdoc_test( else: runtool = ["--runtool=/usr/bin/env"] + plain_env, path_env = _process_env(compile_ctx, ctx.attrs.env, exec_is_windows) + doc_plain_env, doc_path_env = _process_env(compile_ctx, ctx.attrs.doc_env, exec_is_windows) + for k, v in doc_plain_env.items(): + path_env.pop(k, None) + plain_env[k] = v + for k, v in doc_path_env.items(): + plain_env.pop(k, None) + path_env[k] = v + + # `--runtool` is unstable. + plain_env["RUSTC_BOOTSTRAP"] = cmd_args("1") + unstable_options = ["-Zunstable-options"] + rustdoc_cmd = cmd_args( + [cmd_args("--env=", k, "=", v, delimiter = "") for k, v in plain_env.items()], + [cmd_args("--path-env=", k, "=", v, delimiter = "") for k, v in path_env.items()], + toolchain_info.rustdoc, "--test", - "-Zunstable-options", + unstable_options, cmd_args("--test-builder=", toolchain_info.compiler, delimiter = ""), toolchain_info.rustdoc_flags, ctx.attrs.rustdoc_flags, common_args.args, - extern_arg(ctx, compile_ctx, [], attr_crate(ctx), library.rlib), + extern_arg([], attr_crate(ctx), rlib), "--extern=proc_macro" if ctx.attrs.proc_macro else [], compile_ctx.linker_args, cmd_args(linker_argsfile, format = "-Clink-arg=@{}"), @@ -327,26 +350,13 @@ def generate_rustdoc_test( executable_args.runtime_files, ) - rustdoc_cmd = _long_command( + return _long_command( ctx = ctx, - exe = toolchain_info.rustdoc, + exe = toolchain_info.rustc_action, args = rustdoc_cmd, argfile_name = "{}.args".format(common_args.subdir), ) - plain_env, path_env = _process_env(compile_ctx, ctx.attrs.env, exec_is_windows) - rustdoc_env = plain_env | path_env - - # Pass everything in env + doc_env, except ones with value None in doc_env. - for k, v in ctx.attrs.doc_env.items(): - if v == None: - rustdoc_env.pop(k, None) - else: - rustdoc_env[k] = cmd_args(v) - rustdoc_env["RUSTC_BOOTSTRAP"] = cmd_args("1") # for `-Zunstable-options` - - return (rustdoc_cmd, rustdoc_env) - # Generate multiple compile artifacts so that distinct sets of artifacts can be # generated concurrently. def rust_compile_multi( @@ -354,7 +364,6 @@ def rust_compile_multi( compile_ctx: CompileContext, emits: list[Emit], params: BuildParams, - dep_link_style: LinkStyle, default_roots: list[str], extra_link_args: list[typing.Any] = [], predeclared_outputs: dict[Emit, Artifact] = {}, @@ -370,7 +379,6 @@ def rust_compile_multi( compile_ctx = compile_ctx, emit = emit, params = params, - dep_link_style = dep_link_style, default_roots = default_roots, extra_link_args = extra_link_args, predeclared_outputs = predeclared_outputs, @@ -391,7 +399,6 @@ def rust_compile( compile_ctx: CompileContext, emit: Emit, params: BuildParams, - dep_link_style: LinkStyle, default_roots: list[str], extra_link_args: list[typing.Any] = [], predeclared_outputs: dict[Emit, Artifact] = {}, @@ -411,7 +418,6 @@ def rust_compile( dep_ctx = compile_ctx.dep_ctx, emit = emit, params = params, - dep_link_style = dep_link_style, default_roots = default_roots, is_rustdoc_test = False, ) @@ -422,7 +428,6 @@ def rust_compile( lints, # Report unused --extern crates in the notification stream. ["--json=unused-externs-silent", "-Wunused-crate-dependencies"] if toolchain_info.report_unused_deps else [], - "--json=artifacts", # only needed for pipeline but no harm in always leaving it enabled common_args.args, cmd_args("--remap-path-prefix=", compile_ctx.symlinked_srcs, path_sep, "=", ctx.label.path, path_sep, delimiter = ""), compile_ctx.linker_args, @@ -436,7 +441,7 @@ def rust_compile( # use the predeclared one as the output after the failure filter action # below. Otherwise we'll use the predeclared outputs directly. if toolchain_info.failure_filter: - emit_output, emit_args, extra_out = _rustc_emit( + emit_op = _rustc_emit( ctx = ctx, compile_ctx = compile_ctx, emit = emit, @@ -445,7 +450,7 @@ def rust_compile( params = params, ) else: - emit_output, emit_args, extra_out = _rustc_emit( + emit_op = _rustc_emit( ctx = ctx, compile_ctx = compile_ctx, emit = emit, @@ -475,8 +480,7 @@ def rust_compile( ctx, compile_ctx.dep_ctx, ), - # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy. - to_link_strategy(dep_link_style), + params.dep_link_strategy, ) link_args_output = make_link_args( @@ -487,7 +491,7 @@ def rust_compile( inherited_link_args, ], "{}-{}".format(subdir, tempfile), - output_short_path = emit_output.short_path, + output_short_path = emit_op.output.short_path, ) linker_argsfile, _ = ctx.actions.write( "{}/__{}_linker_args.txt".format(subdir, tempfile), @@ -504,20 +508,20 @@ def rust_compile( ctx = ctx, compile_ctx = compile_ctx, prefix = "{}/{}".format(common_args.subdir, common_args.tempfile), - rustc_cmd = cmd_args(toolchain_info.compiler, rustc_cmd, emit_args), + rustc_cmd = cmd_args(toolchain_info.compiler, rustc_cmd, emit_op.args), diag = "diag", - required_outputs = [emit_output], + required_outputs = [emit_op.output], short_cmd = common_args.short_cmd, is_binary = is_binary, allow_cache_upload = allow_cache_upload, crate_map = common_args.crate_map, - only_artifact = "metadata" if toolchain_info.pipelined and emit == Emit("metadata") else None, + env = emit_op.env, ) # Add clippy diagnostic targets for check builds if common_args.is_check: # We don't really need the outputs from this build, just to keep the artifact accounting straight - clippy_out, clippy_emit_args, _extra_out = _rustc_emit( + clippy_emit_op = _rustc_emit( ctx = ctx, compile_ctx = compile_ctx, emit = emit, @@ -525,7 +529,7 @@ def rust_compile( subdir = common_args.subdir + "-clippy", params = params, ) - clippy_env = dict() + clippy_env = clippy_emit_op.env if toolchain_info.clippy_toml: # Clippy wants to be given a path to a directory containing a # clippy.toml (or .clippy.toml). Our buckconfig accepts an arbitrary @@ -542,10 +546,10 @@ def rust_compile( compile_ctx = compile_ctx, prefix = "{}/{}".format(common_args.subdir, common_args.tempfile), # Lints go first to allow other args to override them. - rustc_cmd = cmd_args(compile_ctx.clippy_wrapper, clippy_lints, rustc_cmd, clippy_emit_args), + rustc_cmd = cmd_args(compile_ctx.clippy_wrapper, clippy_lints, rustc_cmd, clippy_emit_op.args), env = clippy_env, diag = "clippy", - required_outputs = [clippy_out], + required_outputs = [clippy_emit_op.output], short_cmd = common_args.short_cmd, is_binary = False, allow_cache_upload = False, @@ -560,7 +564,7 @@ def rust_compile( stderr = diag["diag.txt"] filter_prov = RustFailureFilter( buildstatus = build_status, - required = emit_output, + required = emit_op.output, stderr = stderr, ) @@ -573,26 +577,42 @@ def rust_compile( short_cmd = common_args.short_cmd, ) else: - filtered_output = emit_output + filtered_output = emit_op.output split_debug_mode = compile_ctx.cxx_toolchain_info.split_debug_mode or SplitDebugMode("none") if emit == Emit("link") and split_debug_mode != SplitDebugMode("none"): - dwo_output_directory = extra_out - external_debug_info = inherited_external_debug_info( + dwo_output_directory = emit_op.extra_out + + # staticlibs and cdylibs are "bundled" in the sense that they are used + # without their dependencies by the rest of the rules. This is normally + # correct, except that the split debuginfo rustc emits for these crate + # types is not bundled. This is arguably inconsistent behavior from + # rustc, but in any case, it means we need to do this bundling manually + # by collecting all the external debuginfo from dependencies + if params.crate_type == CrateType("cdylib") or params.crate_type == CrateType("staticlib"): + extra_external_debug_info = inherited_rust_external_debug_info( + ctx = ctx, + dep_ctx = compile_ctx.dep_ctx, + link_strategy = params.dep_link_strategy, + ) + else: + extra_external_debug_info = [] + all_external_debug_info = inherited_external_debug_info( ctx = ctx, dep_ctx = compile_ctx.dep_ctx, dwo_output_directory = dwo_output_directory, - dep_link_style = params.dep_link_style, + dep_link_strategy = params.dep_link_strategy, ) - dwp_inputs.extend(project_artifacts(ctx.actions, [external_debug_info])) + dwp_inputs.extend(project_artifacts(ctx.actions, [all_external_debug_info])) else: dwo_output_directory = None + extra_external_debug_info = [] if is_binary and dwp_available(compile_ctx.cxx_toolchain_info): dwp_output = dwp( ctx, compile_ctx.cxx_toolchain_info, - emit_output, + emit_op.output, identifier = "{}/__{}_{}_dwp".format(common_args.subdir, common_args.tempfile, str(emit)), category_suffix = "rust", # TODO(T110378142): Ideally, referenced objects are a list of @@ -604,12 +624,24 @@ def rust_compile( else: dwp_output = None + stripped_output = strip_debug_info( + ctx, + paths.join(common_args.subdir, "stripped", output_filename( + attr_simple_crate_for_filenames(ctx), + Emit("link"), + params, + )), + filtered_output, + ) + return RustcOutput( output = filtered_output, + stripped_output = stripped_output, diag = diag, pdb = pdb_artifact, dwp_output = dwp_output, dwo_output_directory = dwo_output_directory, + extra_external_debug_info = extra_external_debug_info, ) # --extern = for direct dependencies @@ -625,7 +657,7 @@ def dependency_args( deps: list[RustDependency], subdir: str, crate_type: CrateType, - dep_link_style: LinkStyle, + dep_link_strategy: LinkStrategy, is_check: bool, is_rustdoc_test: bool) -> (cmd_args, list[(CrateName, Label)]): args = cmd_args() @@ -641,25 +673,38 @@ def dependency_args( else: crate = dep.info.crate - style = style_info(dep.info, dep_link_style) + strategy = strategy_info(dep.info, dep_link_strategy) - use_rmeta = is_check or (compile_ctx.toolchain_info.pipelined and not crate_type_codegen(crate_type) and not is_rustdoc_test) + # With `advanced_unstable_linking`, we unconditionally pass the metadata + # artifacts. There are two things that work together to make this possible + # in the case of binaries: + # + # 1. The actual rlibs appear in the link providers, so they'll still be + # available for the linker to link in + # 2. The metadata artifacts aren't rmetas, but rather rlibs that just + # don't contain any generated code. Rustc can't distinguish these + # from real rlibs, and so doesn't throw an error + # + # The benefit of doing this is that there's no requirment that the + # dependency's generated code be provided to the linker via an rlib. It + # could be provided by other means, say, a link group + use_rmeta = is_check or compile_ctx.dep_ctx.advanced_unstable_linking or (compile_ctx.toolchain_info.pipelined and not crate_type_codegen(crate_type) and not is_rustdoc_test) # Use rmeta dependencies whenever possible because they # should be cheaper to produce. if use_rmeta: - artifact = style.rmeta - transitive_artifacts = style.transitive_rmeta_deps + artifact = strategy.rmeta + transitive_artifacts = strategy.transitive_rmeta_deps else: - artifact = style.rlib - transitive_artifacts = style.transitive_deps + artifact = strategy.rlib + transitive_artifacts = strategy.transitive_deps - for marker in style.transitive_proc_macro_deps.keys(): + for marker in strategy.transitive_proc_macro_deps.keys(): info = available_proc_macros[marker.label][RustLinkInfo] - style = style_info(info, dep_link_style) - transitive_deps[style.rmeta if use_rmeta else style.rlib] = info.crate + strategy = strategy_info(info, dep_link_strategy) + transitive_deps[strategy.rmeta if use_rmeta else strategy.rlib] = info.crate - args.add(extern_arg(ctx, compile_ctx, dep.flags, crate, artifact)) + args.add(extern_arg(dep.flags, crate, artifact)) crate_targets.append((crate, dep.label)) # Because deps of this *target* can also be transitive deps of this compiler @@ -712,18 +757,33 @@ def dynamic_symlinked_dirs( artifacts: dict[Artifact, CrateName]) -> cmd_args: name = "{}-dyn".format(prefix) transitive_dependency_dir = ctx.actions.declare_output(name, dir = True) - do_symlinks = cmd_args( - compile_ctx.toolchain_info.transitive_dependency_symlinks_tool, - cmd_args(transitive_dependency_dir.as_output(), format = "--out-dir={}"), + + # Pass the list of rlibs to transitive_dependency_symlinks.py through a file + # because there can be a lot of them. This avoids running out of command + # line length, particularly on Windows. + relative_path = lambda artifact: (cmd_args(artifact, delimiter = "") + .relative_to(transitive_dependency_dir.project("i")) + .ignore_artifacts()) + artifacts_json = ctx.actions.write_json( + ctx.actions.declare_output("{}-dyn.json".format(prefix)), + [ + (relative_path(artifact), crate.dynamic) + for artifact, crate in artifacts.items() + ], + with_inputs = True, + pretty = True, ) - for artifact, crate in artifacts.items(): - relative_path = cmd_args(artifact).relative_to(transitive_dependency_dir.project("i")) - do_symlinks.add("--artifact", crate.dynamic, relative_path.ignore_artifacts()) + ctx.actions.run( - do_symlinks, + [ + compile_ctx.toolchain_info.transitive_dependency_symlinks_tool, + cmd_args(transitive_dependency_dir.as_output(), format = "--out-dir={}"), + cmd_args(artifacts_json, format = "--artifacts={}"), + ], category = "tdep_symlinks", identifier = str(len(compile_ctx.transitive_dependency_dirs)), ) + compile_ctx.transitive_dependency_dirs[transitive_dependency_dir] = None return cmd_args(transitive_dependency_dir, format = "@{}/dirs").hidden(artifacts.keys()) @@ -767,7 +827,6 @@ def _compute_common_args( dep_ctx: DepCollectionContext, emit: Emit, params: BuildParams, - dep_link_style: LinkStyle, default_roots: list[str], is_rustdoc_test: bool) -> CommonArgsInfo: exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows" @@ -775,18 +834,12 @@ def _compute_common_args( crate_type = params.crate_type - args_key = (crate_type, emit, dep_link_style, is_rustdoc_test) - if False: - # TODO(nga): following `if args_key in ...` is no-op, and typechecker does not like it. - def unknown(): - pass - - args_key = unknown() + args_key = (crate_type, emit, params.dep_link_strategy, is_rustdoc_test) if args_key in compile_ctx.common_args: return compile_ctx.common_args[args_key] # Keep filenames distinct in per-flavour subdirs - subdir = "{}-{}-{}-{}".format(crate_type.value, params.reloc_model.value, dep_link_style.value, emit.value) + subdir = "{}-{}-{}-{}".format(crate_type.value, params.reloc_model.value, params.dep_link_strategy.value, emit.value) if is_rustdoc_test: subdir = "{}-rustdoc-test".format(subdir) @@ -808,7 +861,7 @@ def _compute_common_args( deps = resolve_rust_deps(ctx, dep_ctx), subdir = subdir, crate_type = crate_type, - dep_link_style = dep_link_style, + dep_link_strategy = params.dep_link_strategy, is_check = is_check, is_rustdoc_test = is_rustdoc_test, ) @@ -816,7 +869,7 @@ def _compute_common_args( if crate_type == CrateType("proc-macro"): dep_args.add("--extern=proc_macro") - if crate_type == CrateType("cdylib") or crate_type == CrateType("dylib") and not is_check: + if crate_type in [CrateType("cdylib"), CrateType("dylib")] and not is_check: linker_info = compile_ctx.cxx_toolchain_info.linker_info shlib_name = get_default_shared_library_name(linker_info, ctx.label) dep_args.add(cmd_args( @@ -1014,6 +1067,13 @@ def _crate_root( fail("Could not infer crate_root. candidates=%s\nAdd 'crate_root = \"src/example.rs\"' to your attributes to disambiguate." % candidates.list()) +EmitOperation = record( + output = field(Artifact), + args = field(cmd_args), + env = field(dict[str, str]), + extra_out = field(Artifact | None), +) + # Take a desired output and work out how to convince rustc to generate it def _rustc_emit( ctx: AnalysisContext, @@ -1021,54 +1081,68 @@ def _rustc_emit( emit: Emit, predeclared_outputs: dict[Emit, Artifact], subdir: str, - params: BuildParams) -> (Artifact, cmd_args, [Artifact, None]): + params: BuildParams) -> EmitOperation: toolchain_info = compile_ctx.toolchain_info simple_crate = attr_simple_crate_for_filenames(ctx) crate_type = params.crate_type - # Metadata for pipelining needs has enough info to be used as an input for - # dependents. To do this reliably, follow Cargo's pattern of always doing - # --emit metadata,link, but only using the output we actually need. + # Metadata for pipelining needs has enough info to be used as an input + # for dependents. To do this reliably, we actually emit "link" but + # suppress actual codegen with -Zno-codegen. # # We don't bother to do this with "codegen" crates - ie, ones which are - # linked into an artifact like binaries and dylib, since they're not used as - # a pipelined dependency input. - pipeline_artifact = toolchain_info.pipelined and \ - emit in (Emit("metadata"), Emit("link")) and \ - not crate_type_codegen(crate_type) + # linked into an artifact like binaries and dylib, since they're not + # used as a pipelined dependency input. + pipeline_meta = emit == Emit("metadata") and \ + toolchain_info.pipelined and \ + not crate_type_codegen(crate_type) emit_args = cmd_args() + emit_env = {} + extra_out = None + if emit in predeclared_outputs: emit_output = predeclared_outputs[emit] else: extra_hash = "-" + _metadata(ctx.label, False)[1] emit_args.add("-Cextra-filename={}".format(extra_hash)) - filename = subdir + "/" + output_filename(simple_crate, emit, params, extra_hash) + if pipeline_meta: + # Make sure hollow rlibs are distinct from real ones + filename = subdir + "/hollow/" + output_filename(simple_crate, Emit("link"), params, extra_hash) + else: + filename = subdir + "/" + output_filename(simple_crate, emit, params, extra_hash) emit_output = ctx.actions.declare_output(filename) - # For pipelined builds if we're emitting either metadata or link then make - # sure we generate both and take the one we want. - if pipeline_artifact: - metaext = "" if emit == Emit("metadata") else "_unwanted" - linkext = "" if emit == Emit("link") else "_unwanted" - - emit_args.add( - cmd_args("--emit=metadata=", emit_output.as_output(), metaext, delimiter = ""), - cmd_args("--emit=link=", emit_output.as_output(), linkext, delimiter = ""), - ) - elif emit == Emit("expand"): + if emit == Emit("expand"): + emit_env["RUSTC_BOOTSTRAP"] = "1" emit_args.add( "-Zunpretty=expanded", cmd_args(emit_output.as_output(), format = "-o{}"), ) else: - # Assume https://github.com/rust-lang/rust/issues/85356 is fixed (ie - # https://github.com/rust-lang/rust/pull/85362 is applied) - emit_args.add(cmd_args("--emit=", emit.value, "=", emit_output.as_output(), delimiter = "")) + if toolchain_info.pipelined: + # Even though the unstable flag only appears on one of the branches, we need + # an identical environment between the `-Zno-codegen` and non-`-Zno-codegen` + # command or else there are "found possibly newer version of crate" errors. + emit_env["RUSTC_BOOTSTRAP"] = "1" + + if pipeline_meta: + # If we're doing a pipelined build, instead of emitting an actual rmeta + # we emit a "hollow" .rlib - ie, it only contains lib.rmeta and no object + # code. It should contain full information needed by any dependent + # crate which is generating code (MIR, etc). + # + # IMPORTANT: this flag is the only way that the Emit("metadata") and + # Emit("link") operations are allowed to diverge without causing them to + # get different crate hashes. + emit_args.add("-Zno-codegen") + effective_emit = Emit("link") + else: + effective_emit = emit + + emit_args.add(cmd_args("--emit=", effective_emit.value, "=", emit_output.as_output(), delimiter = "")) - extra_out = None - if emit != Emit("expand"): # Strip file extension from directory name. base, _ext = paths.split_extension(output_filename(simple_crate, emit, params)) extra_dir = subdir + "/extras/" + base @@ -1081,7 +1155,12 @@ def _rustc_emit( incremental_cmd = cmd_args(incremental_out.as_output(), format = "-Cincremental={}") emit_args.add(incremental_cmd) - return (emit_output, emit_args, extra_out) + return EmitOperation( + output = emit_output, + args = emit_args, + env = emit_env, + extra_out = extra_out, + ) # Invoke rustc and capture outputs def _rustc_invoke( @@ -1095,8 +1174,7 @@ def _rustc_invoke( is_binary: bool, allow_cache_upload: bool, crate_map: list[(CrateName, Label)], - env: dict[str, [ResolvedStringWithMacros, Artifact]] = {}, - only_artifact: [None, str] = None) -> (dict[str, Artifact], [Artifact, None]): + env: dict[str, str | ResolvedStringWithMacros | Artifact]) -> (dict[str, Artifact], [Artifact, None]): exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows" toolchain_info = compile_ctx.toolchain_info @@ -1118,11 +1196,8 @@ def _rustc_invoke( "--buck-target={}".format(ctx.label.raw_target()), ) - if only_artifact: - compile_cmd.add("--only-artifact=" + only_artifact) - for k, v in crate_map: - compile_cmd.add(crate_map_arg(ctx, compile_ctx, k, v)) + compile_cmd.add(crate_map_arg(k, v)) for k, v in plain_env.items(): compile_cmd.add(cmd_args("--env=", k, "=", v, delimiter = "")) for k, v in path_env.items(): @@ -1191,7 +1266,7 @@ _ESCAPED_NEWLINE_RE = regex("\\n") # path and non-path content, but we'll burn that bridge when we get to it.) def _process_env( compile_ctx: CompileContext, - env: dict[str, [ResolvedStringWithMacros, Artifact]], + env: dict[str, str | ResolvedStringWithMacros | Artifact], exec_is_windows: bool) -> (dict[str, cmd_args], dict[str, cmd_args]): # Values with inputs (ie artifact references). path_env = {} diff --git a/prelude/rust/build_params.bzl b/prelude/rust/build_params.bzl index 0f6b09ebfb91a..d0cdc8f36f151 100644 --- a/prelude/rust/build_params.bzl +++ b/prelude/rust/build_params.bzl @@ -9,8 +9,8 @@ load( "@prelude//linking:link_info.bzl", - "LinkStyle", - "Linkage", # @unused Used as a type + "LibOutputStyle", + "LinkStrategy", ) load("@prelude//os_lookup:defs.bzl", "OsLookup") load("@prelude//utils:expect.bzl", "expect") @@ -41,10 +41,6 @@ def crate_type_native_linkage(crate_type: CrateType) -> bool: def crate_type_linked(crate_type: CrateType) -> bool: return crate_type.value in ("bin", "dylib", "proc-macro", "cdylib") -# Crate type which should include transitive deps -def crate_type_transitive_deps(crate_type: CrateType) -> bool: - return crate_type.value in ("rlib", "dylib", "staticlib") # not sure about staticlib - # Crate type which should always need codegen def crate_type_codegen(crate_type: CrateType) -> bool: return crate_type_linked(crate_type) or crate_type_native_linkage(crate_type) @@ -84,8 +80,7 @@ def emit_needs_codegen(emit: Emit) -> bool: BuildParams = record( crate_type = field(CrateType), reloc_model = field(RelocModel), - # TODO(cjhopman): Is this a LibOutputStyle or a LinkStrategy? - dep_link_style = field(LinkStyle), # what link_style to use for dependencies + dep_link_strategy = field(LinkStrategy), # A prefix and suffix to use for the name of the produced artifact. Note that although we store # these in this type, they are in principle computable from the remaining fields and the OS. # Keeping them here just turns out to be a little more convenient. @@ -95,8 +90,6 @@ BuildParams = record( RustcFlags = record( crate_type = field(CrateType), - reloc_model = field(RelocModel), - dep_link_style = field(LinkStyle), platform_to_affix = field(typing.Callable), ) @@ -156,9 +149,7 @@ LinkageLang = enum( "native-unbundled", ) -_BINARY_SHARED = 0 -_BINARY_PIE = 1 -_BINARY_NON_PIE = 2 +_BINARY = 0 _NATIVE_LINKABLE_SHARED_OBJECT = 3 _RUST_DYLIB_SHARED = 4 _RUST_PROC_MACRO = 5 @@ -184,153 +175,152 @@ def _library_prefix_suffix(linker_type: str, target_os_type: OsLookup) -> (str, }[linker_type] _BUILD_PARAMS = { - _BINARY_SHARED: RustcFlags( - crate_type = CrateType("bin"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("shared"), - platform_to_affix = _executable_prefix_suffix, - ), - _BINARY_PIE: RustcFlags( + _BINARY: RustcFlags( crate_type = CrateType("bin"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("static_pic"), - platform_to_affix = _executable_prefix_suffix, - ), - _BINARY_NON_PIE: RustcFlags( - crate_type = CrateType("bin"), - reloc_model = RelocModel("static"), - dep_link_style = LinkStyle("static"), platform_to_affix = _executable_prefix_suffix, ), _NATIVE_LINKABLE_SHARED_OBJECT: RustcFlags( crate_type = CrateType("cdylib"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("shared"), platform_to_affix = _library_prefix_suffix, ), _RUST_DYLIB_SHARED: RustcFlags( crate_type = CrateType("dylib"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("shared"), platform_to_affix = _library_prefix_suffix, ), _RUST_PROC_MACRO: RustcFlags( crate_type = CrateType("proc-macro"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("static_pic"), platform_to_affix = _library_prefix_suffix, ), + # FIXME(JakobDegen): Add a comment explaining why `.a`s need reloc-strategy + # dependent names while `.rlib`s don't. _RUST_STATIC_PIC_LIBRARY: RustcFlags( crate_type = CrateType("rlib"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("static_pic"), platform_to_affix = lambda _l, _t: ("lib", ".rlib"), ), _RUST_STATIC_NON_PIC_LIBRARY: RustcFlags( crate_type = CrateType("rlib"), - reloc_model = RelocModel("static"), - dep_link_style = LinkStyle("static"), platform_to_affix = lambda _l, _t: ("lib", ".rlib"), ), _NATIVE_LINKABLE_STATIC_PIC: RustcFlags( crate_type = CrateType("staticlib"), - reloc_model = RelocModel("pic"), - dep_link_style = LinkStyle("static_pic"), platform_to_affix = lambda _l, _t: ("lib", "_pic.a"), ), _NATIVE_LINKABLE_STATIC_NON_PIC: RustcFlags( crate_type = CrateType("staticlib"), - reloc_model = RelocModel("static"), - dep_link_style = LinkStyle("static"), platform_to_affix = lambda _l, _t: ("lib", ".a"), ), } _INPUTS = { - # Binary, shared - ("binary", False, "shared", "any", "rust"): _BINARY_SHARED, - ("binary", False, "shared", "shared", "rust"): _BINARY_SHARED, - ("binary", False, "shared", "static", "rust"): _BINARY_SHARED, - # Binary, PIE - ("binary", False, "static_pic", "any", "rust"): _BINARY_PIE, - ("binary", False, "static_pic", "shared", "rust"): _BINARY_PIE, - ("binary", False, "static_pic", "static", "rust"): _BINARY_PIE, - # Binary, non-PIE - ("binary", False, "static", "any", "rust"): _BINARY_NON_PIE, - ("binary", False, "static", "shared", "rust"): _BINARY_NON_PIE, - ("binary", False, "static", "static", "rust"): _BINARY_NON_PIE, + # Binary + ("binary", False, None, "rust"): _BINARY, # Native linkable shared object - ("library", False, "shared", "any", "native"): _NATIVE_LINKABLE_SHARED_OBJECT, - ("library", False, "shared", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT, - ("library", False, "static", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT, - ("library", False, "static_pic", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT, + ("library", False, "shared_lib", "native"): _NATIVE_LINKABLE_SHARED_OBJECT, # Native unbundled linkable shared object - ("library", False, "shared", "any", "native-unbundled"): _RUST_DYLIB_SHARED, - ("library", False, "shared", "shared", "native-unbundled"): _RUST_DYLIB_SHARED, - ("library", False, "static", "shared", "native-unbundled"): _RUST_DYLIB_SHARED, - ("library", False, "static_pic", "shared", "native-unbundled"): _RUST_DYLIB_SHARED, + ("library", False, "shared_lib", "native-unbundled"): _RUST_DYLIB_SHARED, # Rust dylib shared object - ("library", False, "shared", "any", "rust"): _RUST_DYLIB_SHARED, - ("library", False, "shared", "shared", "rust"): _RUST_DYLIB_SHARED, - ("library", False, "static", "shared", "rust"): _RUST_DYLIB_SHARED, - ("library", False, "static_pic", "shared", "rust"): _RUST_DYLIB_SHARED, + ("library", False, "shared_lib", "rust"): _RUST_DYLIB_SHARED, # Rust proc-macro - ("library", True, "shared", "any", "rust"): _RUST_PROC_MACRO, - ("library", True, "shared", "shared", "rust"): _RUST_PROC_MACRO, - ("library", True, "shared", "static", "rust"): _RUST_PROC_MACRO, - ("library", True, "static", "any", "rust"): _RUST_PROC_MACRO, - ("library", True, "static", "shared", "rust"): _RUST_PROC_MACRO, - ("library", True, "static", "static", "rust"): _RUST_PROC_MACRO, - ("library", True, "static_pic", "any", "rust"): _RUST_PROC_MACRO, - ("library", True, "static_pic", "shared", "rust"): _RUST_PROC_MACRO, - ("library", True, "static_pic", "static", "rust"): _RUST_PROC_MACRO, + ("library", True, "archive", "rust"): _RUST_PROC_MACRO, + ("library", True, "pic_archive", "rust"): _RUST_PROC_MACRO, + ("library", True, "shared_lib", "rust"): _RUST_PROC_MACRO, # Rust static_pic library - ("library", False, "shared", "static", "rust"): _RUST_STATIC_PIC_LIBRARY, - ("library", False, "static_pic", "any", "rust"): _RUST_STATIC_PIC_LIBRARY, - ("library", False, "static_pic", "static", "rust"): _RUST_STATIC_PIC_LIBRARY, + ("library", False, "pic_archive", "rust"): _RUST_STATIC_PIC_LIBRARY, # Rust static (non-pic) library - ("library", False, "static", "any", "rust"): _RUST_STATIC_NON_PIC_LIBRARY, - ("library", False, "static", "static", "rust"): _RUST_STATIC_NON_PIC_LIBRARY, + ("library", False, "archive", "rust"): _RUST_STATIC_NON_PIC_LIBRARY, # Native linkable static_pic - ("library", False, "shared", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC, - ("library", False, "static_pic", "any", "native"): _NATIVE_LINKABLE_STATIC_PIC, - ("library", False, "static_pic", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC, + ("library", False, "pic_archive", "native"): _NATIVE_LINKABLE_STATIC_PIC, # Native linkable static non-pic - ("library", False, "static", "any", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC, - ("library", False, "static", "static", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC, + ("library", False, "archive", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC, # Native Unbundled static_pic library - ("library", False, "shared", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY, - ("library", False, "static_pic", "any", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY, - ("library", False, "static_pic", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY, + ("library", False, "pic_archive", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY, # Native Unbundled static (non-pic) library - ("library", False, "static", "any", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY, - ("library", False, "static", "static", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY, + ("library", False, "archive", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY, } # Check types of _INPUTS, writing these out as types is too verbose, but let's make sure we don't have any typos. [ - (RuleType(rule_type), LinkStyle(link_style), Linkage(preferred_linkage), LinkageLang(linkage_lang)) - for (rule_type, _, link_style, preferred_linkage, linkage_lang), _ in _INPUTS.items() + (RuleType(rule_type), LibOutputStyle(lib_output_style) if lib_output_style else None, LinkageLang(linkage_lang)) + for (rule_type, _, lib_output_style, linkage_lang), _ in _INPUTS.items() ] -def _get_flags(build_kind_key: int, target_os_type: OsLookup) -> (RustcFlags, RelocModel): - flags = _BUILD_PARAMS[build_kind_key] - - # On Windows we should always use pic reloc model. +def _get_reloc_model(link_strategy: LinkStrategy, target_os_type: OsLookup) -> RelocModel: if target_os_type.platform == "windows": - return flags, RelocModel("pic") - return flags, flags.reloc_model + return RelocModel("pic") + if link_strategy == LinkStrategy("static"): + return RelocModel("static") + return RelocModel("pic") -# Compute crate type, relocation model and name mapping given what rule we're building, -# whether its a proc-macro, linkage information and language. +# Compute crate type, relocation model and name mapping given what rule we're building, whether its +# a proc-macro, linkage information and language. +# +# Binaries should pass the link strategy and not the lib output style, while libraries should do the +# opposite. +# +# The linking information that's passed here is different from what one might expect in the C++ +# rules. There's a good reason for that, so let's go over it. First, let's recap how C++ handles +# this, as of December 2023 (I say "recap" but I don't think this is actually documented anywhere): +# +# 1. C++ libraries can be built in three different ways: Archives, pic archives, and shared +# libraries. Which one of these is used for a given link strategy is determined by the preferred +# linkage using `linking/link_info.bzl:get_lib_output_style`. +# 2. When a C++ library is built as a shared library, the link strategy used for its dependencies +# is determined by the link style attribute on the C++ library. +# 3. When a C++ library is built as an archive (either kind), there's no need to know a link +# strategy for the dependencies. None of the per-link-strategy providers of the dependencies +# need to be accessed. +# +# There are two relevant ways in which Rust differs: +# +# 1. There are more ways of building Rust libraries than are represented by `LibOutputStyle`. The +# Rust analogue is the `BuildParams` type, which implicitly holds a `LibOutputStyle` as well as +# a bunch of additional information - this is why `LibOutputStyle` is relatively rarely used +# directly in the Rust rules. +# 2. Rust does not have the property in point three above, ie building a Rust library into an +# archive does require knowing per-link-strategy properties of the dependencies. This is +# fundamental in cases without native unbundled deps - with native unbundled deps it may be +# fixable, but that's not super clear. def build_params( rule: RuleType, proc_macro: bool, - link_style: LinkStyle, - preferred_linkage: Linkage, + link_strategy: LinkStrategy | None, + lib_output_style: LibOutputStyle | None, lang: LinkageLang, linker_type: str, target_os_type: OsLookup) -> BuildParams: + if rule == RuleType("binary"): + expect(link_strategy != None) + expect(lib_output_style == None) + else: + expect(link_strategy == None) + expect(lib_output_style != None) + + # FIXME(JakobDegen): We deal with Rust needing to know the link strategy + # even for building archives by using a default link strategy specifically + # for those cases. I've gone through the code and checked all the places + # where the link strategy is used to determine that this won't do anything + # too bad, but it would be nice to enforce that more strictly or not have + # this at all. + def default_link_strategy_for_output_style(output_style: LibOutputStyle) -> LinkStrategy: + if output_style == LibOutputStyle("archive"): + return LinkStrategy("static") + if output_style == LibOutputStyle("pic_archive"): + return LinkStrategy("static_pic") + + # Rust does not have the `link_style` attribute on libraries in the same + # way that C++ does - if it did, this is what it would affect. + return LinkStrategy("shared") + + if not link_strategy: + if proc_macro: + # FIXME(JakobDegen): It's not really clear what we should do about + # proc macros. The principled thing is probably to treat them sort + # of like a normal library, except that they always have preferred + # linkage shared? Preserve existing behavior for now + link_strategy = LinkStrategy("static_pic") + else: + link_strategy = default_link_strategy_for_output_style(lib_output_style) + if rule == RuleType("binary") and proc_macro: # It's complicated: this is a rustdoc test for a procedural macro crate. # We need deps built as if this were a binary, while passing crate-type @@ -340,26 +330,25 @@ def build_params( else: crate_type = None - input = (rule.value, proc_macro, link_style.value, preferred_linkage.value, lang.value) + input = (rule.value, proc_macro, lib_output_style.value if lib_output_style else None, lang.value) expect( input in _INPUTS, - "missing case for rule_type={} proc_macro={} link_style={} preferred_linkage={} lang={}", + "missing case for rule_type={} proc_macro={} lib_output_style={} lang={}", rule, proc_macro, - link_style, - preferred_linkage, + lib_output_style, lang, ) - build_kind_key = _INPUTS[input] - flags, reloc_model = _get_flags(build_kind_key, target_os_type) + flags = _BUILD_PARAMS[_INPUTS[input]] + reloc_model = _get_reloc_model(link_strategy, target_os_type) prefix, suffix = flags.platform_to_affix(linker_type, target_os_type) return BuildParams( crate_type = crate_type or flags.crate_type, reloc_model = reloc_model, - dep_link_style = flags.dep_link_style, + dep_link_strategy = link_strategy, prefix = prefix, suffix = suffix, ) diff --git a/prelude/rust/cargo_buildscript.bzl b/prelude/rust/cargo_buildscript.bzl index 88c09c23964ae..0b91c80d4c540 100644 --- a/prelude/rust/cargo_buildscript.bzl +++ b/prelude/rust/cargo_buildscript.bzl @@ -20,7 +20,7 @@ load("@prelude//:prelude.bzl", "native") load("@prelude//decls:common.bzl", "buck") -load("@prelude//linking:link_info.bzl", "LinkStyle") +load("@prelude//linking:link_info.bzl", "LinkStrategy") load("@prelude//os_lookup:defs.bzl", "OsLookup") load("@prelude//rust:rust_toolchain.bzl", "RustToolchainInfo") load("@prelude//rust:targets.bzl", "targets") @@ -53,7 +53,7 @@ def _make_rustc_shim(ctx: AnalysisContext, cwd: Artifact) -> cmd_args: deps, "any", # subdir CrateType("rlib"), - LinkStyle("static_pic"), + LinkStrategy("static_pic"), True, # is_check False, # is_rustdoc_test ) diff --git a/prelude/rust/context.bzl b/prelude/rust/context.bzl index 8540e977ab118..6f0ced6bfce16 100644 --- a/prelude/rust/context.bzl +++ b/prelude/rust/context.bzl @@ -6,7 +6,7 @@ # of this source tree. load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") -load("@prelude//linking:link_info.bzl", "LinkStyle") +load("@prelude//linking:link_info.bzl", "LinkStrategy") load(":build_params.bzl", "CrateType", "Emit") load(":rust_toolchain.bzl", "PanicRuntime", "RustExplicitSysrootDeps", "RustToolchainInfo") @@ -26,15 +26,6 @@ CommonArgsInfo = record( crate_map = field(list[(CrateName, Label)]), ) -ExternArg = record( - flags = str, - lib = field(Artifact), -) - -CrateMapArg = record( - label = field(Label), -) - # Information that determines how dependencies should be collected DepCollectionContext = record( advanced_unstable_linking = field(bool), @@ -62,8 +53,6 @@ CompileContext = record( # Clippy wrapper (wrapping clippy-driver so it has the same CLI as rustc). clippy_wrapper = field(cmd_args), # Memoized common args for reuse. - common_args = field(dict[(CrateType, Emit, LinkStyle), CommonArgsInfo]), - flagfiles_for_extern = field(dict[ExternArg, Artifact]), - flagfiles_for_crate_map = field(dict[CrateMapArg, Artifact]), + common_args = field(dict[(CrateType, Emit, LinkStrategy, bool), CommonArgsInfo]), transitive_dependency_dirs = field(dict[Artifact, None]), ) diff --git a/prelude/rust/extern.bzl b/prelude/rust/extern.bzl index 443e4db852e87..d2702ded185a4 100644 --- a/prelude/rust/extern.bzl +++ b/prelude/rust/extern.bzl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load(":context.bzl", "CompileContext", "CrateMapArg", "CrateName", "ExternArg") +load(":context.bzl", "CrateName") # Create `--extern` flag. For crates with a name computed during analysis: # @@ -13,48 +13,22 @@ load(":context.bzl", "CompileContext", "CrateMapArg", "CrateName", "ExternArg") # # For crates with a name computed during build: # -# --extern @extern/libPROVISIONAL +# --extern=$(cat path/to/REALNAME)=path/to/libPROVISIONAL.rlib # -# where extern/libPROVISIONAL holds a flag containing the real crate name: -# -# REALNAME=path/to/libPROVISIONAL.rlib -# -# The `compile_ctx` may be omitted for non-dynamic crate names -def extern_arg( - ctx: AnalysisContext, - compile_ctx: CompileContext | None, - flags: list[str], - crate: CrateName, - lib: Artifact) -> cmd_args: +def extern_arg(flags: list[str], crate: CrateName, lib: Artifact) -> cmd_args: if flags == []: flags = "" else: flags = ",".join(flags) + ":" if crate.dynamic: - args = ExternArg(flags = flags, lib = lib) - flagfile = compile_ctx.flagfiles_for_extern.get(args, None) - if not flagfile: - flagfile = ctx.actions.declare_output("extern/{}".format(lib.short_path)) - concat_cmd = [ - compile_ctx.toolchain_info.concat_tool, - "--output", - flagfile.as_output(), - "--", - flags, - cmd_args("@", crate.dynamic, delimiter = ""), - "=", - cmd_args(lib).ignore_artifacts(), - ] - ctx.actions.run( - concat_cmd, - category = "concat", - identifier = str(len(compile_ctx.flagfiles_for_extern)), - ) - compile_ctx.flagfiles_for_extern[args] = flagfile - return cmd_args("--extern", cmd_args("@", flagfile, delimiter = "")).hidden(lib) + # TODO: consider using `cmd_args(crate.dynamic, quote = "json")` so it + # doesn't fall apart on paths containing ')' + crate_name = cmd_args(crate.dynamic, format = "$(cat {})") else: - return cmd_args("--extern=", flags, crate.simple, "=", lib, delimiter = "") + crate_name = crate.simple + + return cmd_args("--extern=", flags, crate_name, "=", lib, delimiter = "") # Create `--crate-map` flag. For crates with a name computed during analysis: # @@ -62,37 +36,12 @@ def extern_arg( # # For crates with a name computed during build: # -# --crate-map @cratemap/path/to/target +# --crate-map=$(cat path/to/REALNAME)=//path/to:target # -# where cratemap/path/to/target holds a flag containing the real crate name: -# -# REALNAME=//path/to:target -# -def crate_map_arg( - ctx: AnalysisContext, - compile_ctx: CompileContext, - crate: CrateName, - label: Label) -> cmd_args: +def crate_map_arg(crate: CrateName, label: Label) -> cmd_args: if crate.dynamic: - args = CrateMapArg(label = label) - flagfile = compile_ctx.flagfiles_for_crate_map.get(args, None) - if not flagfile: - flagfile = ctx.actions.declare_output("cratemap/{}/{}/{}".format(label.cell, label.package, label.name)) - concat_cmd = [ - compile_ctx.toolchain_info.concat_tool, - "--output", - flagfile.as_output(), - "--", - cmd_args("@", crate.dynamic, delimiter = ""), - "=", - str(label.raw_target()), - ] - ctx.actions.run( - concat_cmd, - category = "cratemap", - identifier = str(len(compile_ctx.flagfiles_for_crate_map)), - ) - compile_ctx.flagfiles_for_crate_map[args] = flagfile - return cmd_args("--crate-map", cmd_args("@", flagfile, delimiter = "")) + crate_name = cmd_args(crate.dynamic, format = "$(cat {})") else: - return cmd_args("--crate-map=", crate.simple, "=", str(label.raw_target()), delimiter = "") + crate_name = crate.simple + + return cmd_args("--crate-map=", crate_name, "=", str(label.raw_target()), delimiter = "") diff --git a/prelude/rust/link_info.bzl b/prelude/rust/link_info.bzl index a3fa9be70e460..e392a74050bec 100644 --- a/prelude/rust/link_info.bzl +++ b/prelude/rust/link_info.bzl @@ -36,15 +36,15 @@ load( load( "@prelude//linking:link_groups.bzl", "LinkGroupLib", # @unused Used as a type + "LinkGroupLibInfo", # @unused Used as a type ) load( "@prelude//linking:link_info.bzl", "LinkInfo", - "LinkStyle", + "LinkStrategy", "Linkage", # @unused Used as a type "MergedLinkInfo", "get_link_args_for_strategy", - "to_link_strategy", "unpack_external_debug_info", ) load( @@ -53,18 +53,10 @@ load( "create_linkable_graph", "get_linkable_graph_node_map_func", ) -load( - "@prelude//linking:linkables.bzl", - "linkables", -) load( "@prelude//linking:shared_libraries.bzl", "SharedLibraryInfo", ) -load( - "@prelude//utils:utils.bzl", - "filter_and_map_idx", -) load( ":context.bzl", "CrateName", # @unused Used as a type @@ -72,8 +64,8 @@ load( ) load(":rust_toolchain.bzl", "PanicRuntime") -# Link style for targets which do not set an explicit `link_style` attribute. -DEFAULT_STATIC_LINK_STYLE = LinkStyle("static_pic") +# Link strategy for targets which do not set an explicit `link_style` attribute. +DEFAULT_STATIC_LINK_STRATEGY = LinkStrategy("static_pic") # Override dylib crates to static_pic, so that Rust code is always # statically linked. @@ -93,7 +85,7 @@ RustProcMacroMarker = provider(fields = { }) # Information which is keyed on link_style -RustLinkStyleInfo = record( +RustLinkStrategyInfo = record( # Path to library or binary rlib = field(Artifact), # Transitive dependencies which are relevant to the consumer. For crate types which do not @@ -120,26 +112,67 @@ RustLinkInfo = provider( fields = { # crate - crate name "crate": CrateName, - # styles - information about each LinkStyle as RustLinkStyleInfo - "styles": dict[LinkStyle, RustLinkStyleInfo], - # Propagate native linkable dependencies through rust libraries. - "exported_link_deps": typing.Any, - # Propagate native linkable info through rust libraries. - "merged_link_info": typing.Any, - # Propagate shared libraries through rust libraries. - "shared_libs": typing.Any, + # strategies - information about each LinkStrategy as RustLinkStrategyInfo + "strategies": dict[LinkStrategy, RustLinkStrategyInfo], + # Rust interacts with the native link graph in a non-standard way. Specifically, imagine we + # have a Rust library `:B` with its only one dependency `:A`, another Rust library. The Rust + # rules give Rust -> Rust dependencies special treatment, and as a result, the + # `MergedLinkInfo` provided from `:B` is not a "superset" of the `MergedLinkInfo` provided + # from `:A` (concrete differences discussed below). + # + # This distinction is implemented by effectively having each Rust library provide two sets + # of link providers. The first is the link providers used across Rust -> Rust dependency + # edges - this is what the fields below are. The second set is the one that is used by C++ + # and other non-Rust dependents, and is returned from the rule like normal. The second set + # is a superset of the first, that is it includes anything that the first link providers + # added. + # + # The way in which the native link providers and Rust link providers differ depends on + # whether `advanced_unstable_linking` is set on the toolchain. + # + # * Without `advanced_unstable_linking`, the Rust `MergedLinkInfo` provided by `:A` is only + # the result of merging the `MergedLinkInfo`s from `:A`'s deps, and does not contain + # anything about `:A`. Instead, when `:B` produces the native `MergedLinkInfo`, it will + # add a single static library that bundles all transitive Rust deps, including `:A` (and + # similarly for the DSO case). + # * With `advanced_unstable_linking`, the Rust `MergedLinkInfo` provided by a `:A` does + # include a linkable from `:A`, however that linkable is always the rlib (a static + # library), regardless of `:A`'s preferred linkage or the link strategy. This matches the + # `FORCE_RLIB` behavior, in which Rust -> Rust dependency edges are always statically + # linked. The native link provider then depends on that, and only adds a linkable for the + # `shared_lib` case. + "merged_link_info": MergedLinkInfo, + "shared_libs": SharedLibraryInfo, + # Because of the weird representation of `LinkableGraph`, there is no + # correct way to merge multiple linkable graphs without adding a new + # node at the same time. So we store a list to be able to depend on more + # than one + "linkable_graphs": list[LinkableGraph], + # LinkGroupLibInfo intentionally omitted because the Rust -> Rust version + # never needs to be different from the Rust -> native version + # + # Rust currently treats all native dependencies as being exported, in + # the sense of C++ `exported_deps`. However, they are not only exported + # from the Rust library that directly depends on them, they are also + # exported through any further chains of Rust libraries. This list + # tracks those dependencies + # + # FIXME(JakobDegen): We should not default to treating all native deps + # as exported. + "exported_link_deps": list[Dependency], }, ) -def _adjust_link_style_for_rust_dependencies(dep_link_style: LinkStyle) -> LinkStyle: - if FORCE_RLIB and dep_link_style == LinkStyle("shared"): - return DEFAULT_STATIC_LINK_STYLE +def _adjust_link_strategy_for_rust_dependencies(dep_link_strategy: LinkStrategy) -> LinkStrategy: + if FORCE_RLIB and dep_link_strategy == LinkStrategy("shared"): + return DEFAULT_STATIC_LINK_STRATEGY else: - return dep_link_style + return dep_link_strategy -def style_info(info: RustLinkInfo, dep_link_style: LinkStyle) -> RustLinkStyleInfo: - rust_dep_link_style = _adjust_link_style_for_rust_dependencies(dep_link_style) - return info.styles[rust_dep_link_style] +def strategy_info(info: RustLinkInfo, dep_link_strategy: LinkStrategy) -> RustLinkStrategyInfo: + rust_dep_link_strategy = _adjust_link_strategy_for_rust_dependencies(dep_link_strategy) + + return info.strategies[rust_dep_link_strategy] # Any dependency of a Rust crate RustOrNativeDependency = record( @@ -179,14 +212,14 @@ RustCxxLinkGroupInfo = record( def enable_link_groups( ctx: AnalysisContext, - link_style: [LinkStyle, None], - specified_link_style: LinkStyle, + link_strategy: [LinkStrategy, None], + specified_link_strategy: LinkStrategy, is_binary: bool): if not (cxx_is_gnu(ctx) and is_binary): # check minium requirements return False - if link_style == LinkStyle("shared") or link_style != specified_link_style: - # check whether we should run link groups analysis for the given link style + if link_strategy == LinkStrategy("shared") or link_strategy != specified_link_strategy: + # check whether we should run link groups analysis for the given link strategy return False # check whether link groups is enabled @@ -313,17 +346,6 @@ def resolve_rust_deps( def get_available_proc_macros(ctx: AnalysisContext) -> dict[TargetLabel, Dependency]: return {x.label.raw_target(): x for x in ctx.plugins[RustProcMacroPlugin]} -def _create_linkable_graph( - ctx: AnalysisContext, - deps: list[Dependency]) -> LinkableGraph: - linkable_graph = create_linkable_graph( - ctx, - deps = filter(None, ( - [d.linkable_graph for d in linkables(deps)] - )), - ) - return linkable_graph - # Returns native link dependencies. def _native_link_dependencies( ctx: AnalysisContext, @@ -337,93 +359,57 @@ def _native_link_dependencies( """ first_order_deps = [dep.dep for dep in resolve_deps(ctx, dep_ctx)] - if dep_ctx.advanced_unstable_linking: - return [d for d in first_order_deps if MergedLinkInfo in d] - else: - return [ - d - for d in first_order_deps - if RustLinkInfo not in d and MergedLinkInfo in d - ] - -# Returns native link dependencies. -def _native_link_infos( - ctx: AnalysisContext, - dep_ctx: DepCollectionContext) -> list[MergedLinkInfo]: - """ - Return all first-order native link infos of all transitive Rust libraries. - """ - link_deps = _native_link_dependencies(ctx, dep_ctx) - return [d[MergedLinkInfo] for d in link_deps] - -# Returns native link dependencies. -def _native_shared_lib_infos( - ctx: AnalysisContext, - dep_ctx: DepCollectionContext) -> list[SharedLibraryInfo]: - """ - Return all transitive shared libraries for non-Rust native linkabes. - - This emulates v1's graph walk, where it traverses through -- and ignores -- - Rust libraries to collect all transitive shared libraries. - """ - first_order_deps = [dep.dep for dep in resolve_deps(ctx, dep_ctx)] - - if dep_ctx.advanced_unstable_linking: - return [d[SharedLibraryInfo] for d in first_order_deps if SharedLibraryInfo in d] - else: - return [ - d[SharedLibraryInfo] - for d in first_order_deps - if RustLinkInfo not in d and SharedLibraryInfo in d - ] + return [ + d + for d in first_order_deps + if RustLinkInfo not in d and MergedLinkInfo in d + ] -# Returns native link dependencies. -def _rust_link_infos( +# Returns the rust link infos for non-proc macro deps. +# +# This is intended to be used to access the Rust -> Rust link providers +def _rust_non_proc_macro_link_infos( ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[RustLinkInfo]: - return [d.info for d in resolve_rust_deps(ctx, dep_ctx)] - -def normalize_crate(label: str) -> str: - return label.replace("-", "_") + return [d.info for d in resolve_rust_deps(ctx, dep_ctx) if d.proc_macro_marker == None] def inherited_exported_link_deps(ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[Dependency]: deps = {} for dep in _native_link_dependencies(ctx, dep_ctx): deps[dep.label] = dep - if not dep_ctx.advanced_unstable_linking: - for info in _rust_link_infos(ctx, dep_ctx): - for dep in info.exported_link_deps: - deps[dep.label] = dep + for info in _rust_non_proc_macro_link_infos(ctx, dep_ctx): + for dep in info.exported_link_deps: + deps[dep.label] = dep return deps.values() def inherited_rust_cxx_link_group_info( ctx: AnalysisContext, dep_ctx: DepCollectionContext, - link_style: [LinkStyle, None] = None) -> RustCxxLinkGroupInfo: - link_deps = inherited_exported_link_deps(ctx, dep_ctx) + link_strategy: [LinkStrategy, None] = None) -> RustCxxLinkGroupInfo: + link_graphs = inherited_linkable_graphs(ctx, dep_ctx) # Assume a rust executable wants to use link groups if a link group map # is present link_group = get_link_group(ctx) - link_group_info = get_link_group_info(ctx, filter_and_map_idx(LinkableGraph, link_deps)) + link_group_info = get_link_group_info(ctx, link_graphs) link_groups = link_group_info.groups link_group_mappings = link_group_info.mappings link_group_preferred_linkage = get_link_group_preferred_linkage(link_groups.values()) auto_link_group_specs = get_auto_link_group_specs(ctx, link_group_info) - linkable_graph = _create_linkable_graph( + linkable_graph = create_linkable_graph( ctx, - link_deps, + deps = link_graphs, ) linkable_graph_node_map = get_linkable_graph_node_map_func(linkable_graph)() executable_deps = [] - for d in link_deps: - if d.label in linkable_graph_node_map: - executable_deps.append(d.label) + for g in link_graphs: + if g.label in linkable_graph_node_map: + executable_deps.append(g.label) else: # handle labels that are mutated by version alias - executable_deps.append(d.get(LinkableGraph).nodes.value.label) + executable_deps.append(g.nodes.value.label) linked_link_groups = create_link_groups( ctx = ctx, @@ -454,12 +440,12 @@ def inherited_rust_cxx_link_group_info( link_groups, link_group_mappings, link_group_preferred_linkage, - pic_behavior = PicBehavior("always_enabled") if link_style == LinkStyle("static_pic") else PicBehavior("supported"), + pic_behavior = PicBehavior("always_enabled") if link_strategy == LinkStrategy("static_pic") else PicBehavior("supported"), link_group_libs = { name: (lib.label, lib.shared_link_infos) for name, lib in link_group_libs.items() }, - link_strategy = to_link_strategy(link_style), + link_strategy = link_strategy, roots = executable_deps, is_executable_link = True, prefer_stripped = False, @@ -482,39 +468,60 @@ def inherited_merged_link_infos( ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[MergedLinkInfo]: infos = [] - infos.extend(_native_link_infos(ctx, dep_ctx)) - if not dep_ctx.advanced_unstable_linking: - infos.extend([d.merged_link_info for d in _rust_link_infos(ctx, dep_ctx) if d.merged_link_info]) + infos.extend([d[MergedLinkInfo] for d in _native_link_dependencies(ctx, dep_ctx)]) + infos.extend([d.merged_link_info for d in _rust_non_proc_macro_link_infos(ctx, dep_ctx) if d.merged_link_info]) return infos def inherited_shared_libs( ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[SharedLibraryInfo]: infos = [] - infos.extend(_native_shared_lib_infos(ctx, dep_ctx)) - if not dep_ctx.advanced_unstable_linking: - infos.extend([d.shared_libs for d in _rust_link_infos(ctx, dep_ctx)]) + infos.extend([d[SharedLibraryInfo] for d in _native_link_dependencies(ctx, dep_ctx)]) + infos.extend([d.shared_libs for d in _rust_non_proc_macro_link_infos(ctx, dep_ctx)]) return infos +def inherited_linkable_graphs(ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[LinkableGraph]: + deps = {} + for d in _native_link_dependencies(ctx, dep_ctx): + g = d.get(LinkableGraph) + if g: + deps[g.label] = g + for info in _rust_non_proc_macro_link_infos(ctx, dep_ctx): + for g in info.linkable_graphs: + deps[g.label] = g + return deps.values() + +def inherited_link_group_lib_infos(ctx: AnalysisContext, dep_ctx: DepCollectionContext) -> list[LinkGroupLibInfo]: + # There are no special Rust -> Rust versions of this provider + deps = {} + for d in resolve_deps(ctx, dep_ctx): + i = d.dep.get(LinkGroupLibInfo) + if i: + deps[d.dep.label] = i + return deps.values() + +def inherited_rust_external_debug_info( + ctx: AnalysisContext, + dep_ctx: DepCollectionContext, + link_strategy: LinkStrategy) -> list[ArtifactTSet]: + return [strategy_info(d.info, link_strategy).external_debug_info for d in resolve_rust_deps(ctx, dep_ctx)] + def inherited_external_debug_info( ctx: AnalysisContext, dep_ctx: DepCollectionContext, dwo_output_directory: [Artifact, None], - dep_link_style: LinkStyle) -> ArtifactTSet: - rust_dep_link_style = _adjust_link_style_for_rust_dependencies(dep_link_style) - non_rust_dep_link_style = dep_link_style - + dep_link_strategy: LinkStrategy) -> ArtifactTSet: inherited_debug_infos = [] inherited_link_infos = [] for d in resolve_deps(ctx, dep_ctx): if RustLinkInfo in d.dep: - inherited_debug_infos.append(d.dep[RustLinkInfo].styles[rust_dep_link_style].external_debug_info) + inherited_debug_infos.append(strategy_info(d.dep[RustLinkInfo], dep_link_strategy).external_debug_info) inherited_link_infos.append(d.dep[RustLinkInfo].merged_link_info) elif MergedLinkInfo in d.dep: inherited_link_infos.append(d.dep[MergedLinkInfo]) - link_args = get_link_args_for_strategy(ctx, inherited_link_infos, to_link_strategy(non_rust_dep_link_style)) + link_args = get_link_args_for_strategy(ctx, inherited_link_infos, dep_link_strategy) inherited_debug_infos.append(unpack_external_debug_info(ctx.actions, link_args)) return make_artifact_tset( @@ -524,6 +531,9 @@ def inherited_external_debug_info( children = inherited_debug_infos, ) +def normalize_crate(label: str) -> str: + return label.replace("-", "_") + def attr_simple_crate_for_filenames(ctx: AnalysisContext) -> str: """ A "good enough" identifier to use in filenames. Buck wants to have filenames diff --git a/prelude/rust/rust-analyzer/resolve_deps.bxl b/prelude/rust/rust-analyzer/resolve_deps.bxl index 483251878a05b..61dd48b896b35 100644 --- a/prelude/rust/rust-analyzer/resolve_deps.bxl +++ b/prelude/rust/rust-analyzer/resolve_deps.bxl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//linking:link_info.bzl", "LinkStyle") +load("@prelude//linking:link_info.bzl", "LinkStrategy") load("@prelude//rust:link_info.bzl", "RustLinkInfo") def materialize(ctx, target): @@ -47,6 +47,12 @@ def _process_target_config(ctx, target, in_workspace, out_dir = None): for test in resolved_attrs.tests: tests.append(test.raw_target()) + # materialize a file containing the dynamic crate name + crate_dynamic = getattr(resolved_attrs, "crate_dynamic", None) + if crate_dynamic: + cratename_artifact = crate_dynamic.get(DefaultInfo).default_outputs[0] + crate_dynamic = ctx.output.ensure(cratename_artifact).abs_path() + # copy over the absolute paths and raw targets into the output copy = {} attrs = target.attrs_eager() @@ -61,6 +67,8 @@ def _process_target_config(ctx, target, in_workspace, out_dir = None): copy["named_deps"] = named_deps elif k == "tests": copy["tests"] = tests + elif k == "crate_dynamic": + copy["crate_dynamic"] = crate_dynamic else: copy[k] = getattr(attrs, k) @@ -100,21 +108,28 @@ def cquery_deps(ctx, top_targets, workspaces, actions): cfg["crate_root"] = thrift["artifact"] out[target.label.raw_target()] = cfg elif "generated_protobuf_library_rust" in labels.value(): - protobuf = materialize_generated_protobufs(ctx, target, actions) - out[target.label.raw_target()] = _process_target_config(ctx, target, in_workspace, out_dir = protobuf.abs_path()) + protobuf_out_dir = materialize_generated_protobufs(ctx, target, actions, seen) + out[target.label.raw_target()] = _process_target_config(ctx, target, in_workspace, protobuf_out_dir) else: out[target.label.raw_target()] = _process_target_config(ctx, target, in_workspace) return out -def materialize_generated_protobufs(ctx, target, actions): +def materialize_generated_protobufs(ctx, target, actions, seen): + """If `target` has a dependency that generates code from protobufs, + materialize the generated code and return the path to the output directory. + """ prost_target = target.attrs_lazy().get("named_deps").value().get("generated_prost_target") t = prost_target.raw_target() analysis = ctx.analysis(t) output = analysis.providers()[DefaultInfo].default_outputs[0] outfile = "{}/{}/{}".format(t.cell, t.package, t.name) - copied = ctx.output.ensure(actions.copy_file(outfile, output)) - return copied + if outfile in seen: + return None + seen[outfile] = () + + copied = ctx.output.ensure(actions.copy_file(outfile, output)) + return copied.abs_path() def materialize_generated_thrift(ctx, target, actions, seen): mapped_srcs = target.attrs_lazy().get("mapped_srcs").value() @@ -132,14 +147,16 @@ def materialize_generated_thrift(ctx, target, actions, seen): else: label = label.raw_target() - copied = actions.copy_file(outfile, artifacts.artifacts()[0]) - copied = ctx.output.ensure(copied) - artifact = { - "artifact": copied.abs_path(), - "label": label, - "mapped_src": mapped_src, - } - out.append(artifact) + if len(artifacts.artifacts()) > 0: + copied = actions.copy_file(outfile, artifacts.artifacts()[0]) + copied = ctx.output.ensure(copied) + artifact = { + "artifact": copied.abs_path(), + "label": label, + "mapped_src": mapped_src, + } + out.append(artifact) + seen[outfile] = () return out @@ -154,7 +171,7 @@ def expand_proc_macros(ctx, targets): proc_macro = getattr(attrs, "proc_macro", False) if proc_macro: analysis = ctx.analysis(target) - rlib = analysis.providers()[RustLinkInfo].styles[LinkStyle("shared")].rlib + rlib = analysis.providers()[RustLinkInfo].strategies[LinkStrategy("shared")].rlib label = target.label.raw_target() out[label] = {"actual": label, "dylib": ctx.output.ensure(rlib).abs_path()} return out diff --git a/prelude/rust/rust_binary.bzl b/prelude/rust/rust_binary.bzl index de3f9baf4ed9b..046d43d0438f5 100644 --- a/prelude/rust/rust_binary.bzl +++ b/prelude/rust/rust_binary.bzl @@ -5,6 +5,10 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load( + "@prelude//:artifact_tset.bzl", + "project_artifacts", +) load( "@prelude//:resources.bzl", "create_resource_db", @@ -31,8 +35,7 @@ load( ) load( "@prelude//linking:link_info.bzl", - "LinkStyle", - "Linkage", + "LinkStrategy", ) load( "@prelude//linking:shared_libraries.bzl", @@ -65,9 +68,10 @@ load( load(":context.bzl", "CompileContext") load( ":link_info.bzl", - "DEFAULT_STATIC_LINK_STYLE", + "DEFAULT_STATIC_LINK_STRATEGY", "attr_simple_crate_for_filenames", "enable_link_groups", + "inherited_external_debug_info", "inherited_rust_cxx_link_group_info", "inherited_shared_libs", ) @@ -96,10 +100,10 @@ def _rust_binary_common( styles = {} dwp_target = None pdb = None - style_param = {} # style -> param + strategy_param = {} # strategy -> param sub_targets = {} - specified_link_style = LinkStyle(ctx.attrs.link_style) if ctx.attrs.link_style else DEFAULT_STATIC_LINK_STYLE + specified_link_strategy = LinkStrategy(ctx.attrs.link_style) if ctx.attrs.link_style else DEFAULT_STATIC_LINK_STRATEGY target_os_type = ctx.attrs._target_os_type[OsLookup] linker_type = compile_ctx.cxx_toolchain_info.linker_info.type @@ -110,20 +114,20 @@ def _rust_binary_common( deps = cxx_attr_deps(ctx), ).values()) - for link_style in LinkStyle: + for link_strategy in LinkStrategy: # Unlike for libraries, there's no possibility of different link styles # resulting in the same build params, so no need to deduplicate. params = build_params( rule = RuleType("binary"), proc_macro = False, - link_style = link_style, - preferred_linkage = Linkage("any"), + link_strategy = link_strategy, + lib_output_style = None, lang = LinkageLang("rust"), linker_type = linker_type, target_os_type = target_os_type, ) - style_param[link_style] = params - name = link_style.value + "/" + output_filename(simple_crate, Emit("link"), params) + strategy_param[link_strategy] = params + name = link_strategy.value + "/" + output_filename(simple_crate, Emit("link"), params) output = ctx.actions.declare_output(name) # Gather and setup symlink tree of transitive shared library deps. @@ -136,11 +140,11 @@ def _rust_binary_common( labels_to_links_map = {} filtered_targets = [] - if enable_link_groups(ctx, link_style, specified_link_style, is_binary = True): + if enable_link_groups(ctx, link_strategy, specified_link_strategy, is_binary = True): rust_cxx_link_group_info = inherited_rust_cxx_link_group_info( ctx, compile_ctx.dep_ctx, - link_style = link_style, + link_strategy = link_strategy, ) link_group_mappings = rust_cxx_link_group_info.link_group_info.mappings link_group_libs = rust_cxx_link_group_info.link_group_libs @@ -152,7 +156,7 @@ def _rust_binary_common( # link style. # XXX need link tree for dylib crates shlib_deps = [] - if link_style == LinkStyle("shared") or rust_cxx_link_group_info != None: + if link_strategy == LinkStrategy("shared") or rust_cxx_link_group_info != None: shlib_deps = inherited_shared_libs(ctx, compile_ctx.dep_ctx) shlib_info = merge_shared_libraries(ctx.actions, deps = shlib_deps) @@ -179,7 +183,7 @@ def _rust_binary_common( # link groups shared libraries link args are directly added to the link command, # we don't have to add them here executable_args = executable_shared_lib_arguments( - ctx.actions, + ctx, compile_ctx.cxx_toolchain_info, output, shared_libs, @@ -193,7 +197,6 @@ def _rust_binary_common( compile_ctx = compile_ctx, emits = [Emit("link"), Emit("metadata")], params = params, - dep_link_style = link_style, default_roots = default_roots, extra_link_args = executable_args.extra_link_args, predeclared_outputs = {Emit("link"): output}, @@ -205,6 +208,15 @@ def _rust_binary_common( args = cmd_args(link.output).hidden(executable_args.runtime_files) extra_targets = [("check", meta.output)] + meta.diag.items() + external_debug_info = project_artifacts( + actions = ctx.actions, + tsets = [inherited_external_debug_info( + ctx, + compile_ctx.dep_ctx, + link.dwo_output_directory, + link_strategy, + )], + ) # If we have some resources, write it to the resources JSON file and add # it and all resources to "runtime_files" so that we make to materialize @@ -223,9 +235,9 @@ def _rust_binary_common( args.hidden(resources_hidden) runtime_files.extend(resources_hidden) - sub_targets_for_link_style = {} + sub_targets_for_link_strategy = {} - sub_targets_for_link_style["shared-libraries"] = [DefaultInfo( + sub_targets_for_link_strategy["shared-libraries"] = [DefaultInfo( default_output = ctx.actions.write_json( name + ".shared-libraries.json", { @@ -244,7 +256,7 @@ def _rust_binary_common( )] if isinstance(executable_args.shared_libs_symlink_tree, Artifact): - sub_targets_for_link_style["rpath-tree"] = [DefaultInfo( + sub_targets_for_link_strategy["rpath-tree"] = [DefaultInfo( default_output = executable_args.shared_libs_symlink_tree, other_outputs = [ lib.output @@ -257,51 +269,50 @@ def _rust_binary_common( )] if rust_cxx_link_group_info: - sub_targets_for_link_style[LINK_GROUP_MAP_DATABASE_SUB_TARGET] = [get_link_group_map_json(ctx, filtered_targets)] + sub_targets_for_link_strategy[LINK_GROUP_MAP_DATABASE_SUB_TARGET] = [get_link_group_map_json(ctx, filtered_targets)] readable_mappings = {} for node, group in link_group_mappings.items(): readable_mappings[group] = readable_mappings.get(group, []) + ["{}//{}:{}".format(node.cell, node.package, node.name)] - sub_targets_for_link_style[LINK_GROUP_MAPPINGS_SUB_TARGET] = [DefaultInfo( + sub_targets_for_link_strategy[LINK_GROUP_MAPPINGS_SUB_TARGET] = [DefaultInfo( default_output = ctx.actions.write_json( name + LINK_GROUP_MAPPINGS_FILENAME_SUFFIX, readable_mappings, ), )] - styles[link_style] = _CompileOutputs( + styles[link_strategy] = _CompileOutputs( link = link.output, args = args, extra_targets = extra_targets, runtime_files = runtime_files, - external_debug_info = executable_args.external_debug_info, - sub_targets = sub_targets_for_link_style, + external_debug_info = executable_args.external_debug_info + external_debug_info, + sub_targets = sub_targets_for_link_strategy, dist_info = DistInfo( shared_libs = shlib_info.set, nondebug_runtime_files = runtime_files, ), ) - if link_style == specified_link_style and link.dwp_output: + if link_strategy == specified_link_strategy and link.dwp_output: dwp_target = link.dwp_output - if link_style == specified_link_style and link.pdb: + if link_strategy == specified_link_strategy and link.pdb: pdb = link.pdb expand = rust_compile( ctx = ctx, compile_ctx = compile_ctx, emit = Emit("expand"), - params = style_param[DEFAULT_STATIC_LINK_STYLE], - dep_link_style = DEFAULT_STATIC_LINK_STYLE, + params = strategy_param[DEFAULT_STATIC_LINK_STRATEGY], default_roots = default_roots, extra_flags = extra_flags, ) - compiled_outputs = styles[specified_link_style] + compiled_outputs = styles[specified_link_strategy] extra_compiled_targets = (compiled_outputs.extra_targets + [ ("doc", generate_rustdoc( ctx = ctx, compile_ctx = compile_ctx, - params = style_param[DEFAULT_STATIC_LINK_STYLE], + params = strategy_param[DEFAULT_STATIC_LINK_STRATEGY], default_roots = default_roots, document_private_items = True, )), diff --git a/prelude/rust/rust_library.bzl b/prelude/rust/rust_library.bzl index f955dfc8531a4..6bf6428cdfe6e 100644 --- a/prelude/rust/rust_library.bzl +++ b/prelude/rust/rust_library.bzl @@ -10,7 +10,6 @@ load( "ArtifactTSet", "make_artifact_tset", ) -load("@prelude//:paths.bzl", "paths") load("@prelude//:resources.bzl", "ResourceInfo", "gather_resources") load( "@prelude//android:android_providers.bzl", @@ -20,7 +19,7 @@ load( "@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info", ) -load("@prelude//cxx:cxx_toolchain_types.bzl", "PicBehavior") +load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") load( "@prelude//cxx:linker.bzl", "PDB_SUB_TARGET", @@ -43,10 +42,9 @@ load( "LinkInfo", "LinkInfos", "LinkStrategy", - "LinkStyle", "Linkage", "LinkedObject", - "MergedLinkInfo", + "MergedLinkInfo", # @unused Used as a type "SharedLibLinkable", "create_merged_link_info", "create_merged_link_info_for_propagation", @@ -56,13 +54,14 @@ load( load( "@prelude//linking:linkable_graph.bzl", "DlopenableLibraryInfo", + "LinkableGraph", # @unused Used as a type "create_linkable_graph", "create_linkable_graph_node", "create_linkable_node", ) load( "@prelude//linking:shared_libraries.bzl", - "SharedLibraryInfo", + "SharedLibraryInfo", # @unused Used as a type "create_shared_libraries", "merge_shared_libraries", ) @@ -80,11 +79,11 @@ load( load( ":build_params.bzl", "BuildParams", # @unused Used as a type + "CrateType", "Emit", "LinkageLang", "RuleType", "build_params", - "crate_type_transitive_deps", ) load( ":context.bzl", @@ -94,18 +93,19 @@ load( ) load( ":link_info.bzl", - "DEFAULT_STATIC_LINK_STYLE", + "DEFAULT_STATIC_LINK_STRATEGY", "RustLinkInfo", - "RustLinkStyleInfo", + "RustLinkStrategyInfo", "RustProcMacroMarker", # @unused Used as a type "attr_crate", "inherited_exported_link_deps", - "inherited_external_debug_info", + "inherited_link_group_lib_infos", + "inherited_linkable_graphs", "inherited_merged_link_infos", "inherited_shared_libs", "resolve_deps", "resolve_rust_deps", - "style_info", + "strategy_info", ) load(":proc_macro_alias.bzl", "rust_proc_macro_alias") load(":resources.bzl", "rust_attr_resources") @@ -131,43 +131,10 @@ def prebuilt_rust_library_impl(ctx: AnalysisContext) -> list[Provider]: panic_runtime = rust_toolchain.panic_runtime, ) - # Rust link provider. - crate = attr_crate(ctx) - styles = {} - for style in LinkStyle: - dep_link_style = style - tdeps, tmetadeps, external_debug_info, tprocmacrodeps = _compute_transitive_deps(ctx, dep_ctx, dep_link_style) - external_debug_info = make_artifact_tset( - actions = ctx.actions, - children = external_debug_info, - ) - styles[style] = RustLinkStyleInfo( - rlib = ctx.attrs.rlib, - transitive_deps = tdeps, - rmeta = ctx.attrs.rlib, - transitive_rmeta_deps = tmetadeps, - transitive_proc_macro_deps = tprocmacrodeps, - pdb = None, - external_debug_info = external_debug_info, - ) + cxx_toolchain = get_cxx_toolchain_info(ctx) + linker_info = cxx_toolchain.linker_info - providers.append( - RustLinkInfo( - crate = crate, - styles = styles, - exported_link_deps = inherited_exported_link_deps(ctx, dep_ctx), - merged_link_info = create_merged_link_info_for_propagation(ctx, inherited_merged_link_infos(ctx, dep_ctx)), - shared_libs = merge_shared_libraries( - ctx.actions, - deps = inherited_shared_libs(ctx, dep_ctx), - ), - ), - ) - - linker_info = get_cxx_toolchain_info(ctx).linker_info - - # Native link provier. - link = LinkInfos( + archive_info = LinkInfos( default = LinkInfo( linkables = [ ArchiveLinkable( @@ -191,50 +158,44 @@ def prebuilt_rust_library_impl(ctx: AnalysisContext) -> list[Provider]: ], ), ) - providers.append( - create_merged_link_info( - ctx, - PicBehavior("supported"), - {output_style: link for output_style in LibOutputStyle}, - exported_deps = [d[MergedLinkInfo] for d in ctx.attrs.deps], - # TODO(agallagher): This matches v1 behavior, but some of these libs - # have prebuilt DSOs which might be usable. - preferred_linkage = Linkage("static"), - ), - ) + link_infos = {LibOutputStyle("archive"): archive_info, LibOutputStyle("pic_archive"): archive_info} - # Native link graph setup. - linkable_graph = create_linkable_graph( + # Rust link provider. + crate = attr_crate(ctx) + strategies = {} + for link_strategy in LinkStrategy: + tdeps, tmetadeps, external_debug_info, tprocmacrodeps = _compute_transitive_deps(ctx, dep_ctx, link_strategy) + external_debug_info = make_artifact_tset( + actions = ctx.actions, + children = external_debug_info, + ) + strategies[link_strategy] = RustLinkStrategyInfo( + rlib = ctx.attrs.rlib, + transitive_deps = tdeps, + rmeta = ctx.attrs.rlib, + transitive_rmeta_deps = tmetadeps, + transitive_proc_macro_deps = tprocmacrodeps, + pdb = None, + external_debug_info = external_debug_info, + ) + + merged_link_info, shared_libs, inherited_graphs, inherited_link_deps = _rust_link_providers( ctx, - node = create_linkable_graph_node( - ctx, - linkable_node = create_linkable_node( - ctx = ctx, - preferred_linkage = Linkage("static"), - exported_deps = ctx.attrs.deps, - link_infos = {output_style: link for output_style in LibOutputStyle}, - default_soname = get_default_shared_library_name(linker_info, ctx.label), - ), + dep_ctx, + cxx_toolchain, + link_infos, + Linkage(ctx.attrs.preferred_linkage), + ) + providers.append( + RustLinkInfo( + crate = crate, + strategies = strategies, + exported_link_deps = inherited_link_deps, + merged_link_info = merged_link_info, + shared_libs = shared_libs, + linkable_graphs = inherited_graphs, ), - deps = ctx.attrs.deps, ) - providers.append(linkable_graph) - - providers.append(merge_link_group_lib_info(deps = ctx.attrs.deps)) - - # FIXME(JakobDegen): I am about 85% confident that this matches what C++ - # does for prebuilt libraries if they don't have a shared variant and have - # preferred linkage static. C++ doesn't require static preferred linkage on - # their prebuilt libraries, and so they incur extra complexity here that we - # don't have to deal with. - # - # However, Rust linking is not the same as C++ linking. If Rust were - # disciplined about its use of `LibOutputStyle`, `Linkage` and - # `LinkStrategy`, then this would at least be no more wrong than what C++ - # does. In the meantime however... - providers.append(SharedLibraryInfo(set = None)) - - providers.append(merge_android_packageable_info(ctx.label, ctx.actions, ctx.attrs.deps)) return providers @@ -262,24 +223,43 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: check_artifacts = {"check": meta.output} check_artifacts.update(meta.diag) - rust_param_artifact[params] = _handle_rust_artifact(ctx, compile_ctx.dep_ctx, params, link, meta) + rust_param_artifact[params] = (link, meta) if LinkageLang("native") in param_lang[params] or LinkageLang("native-unbundled") in param_lang[params]: native_param_artifact[params] = link - # Among {rustdoc, doctests, macro expand}, doctests are the only one which - # cares about linkage. So if there is a required link style set for the - # doctests, reuse those same dependency artifacts for the other build - # outputs where static vs static_pic does not make a difference. + # For doctests, we need to know two things to know how to link them. The + # first is that we need a link strategy, which affects how deps of this + # target are handled if ctx.attrs.doc_link_style: - static_link_style = { - "shared": DEFAULT_STATIC_LINK_STYLE, - "static": LinkStyle("static"), - "static_pic": LinkStyle("static_pic"), + doc_link_strategy = LinkStrategy(ctx.attrs.doc_link_style) + else: + # FIXME(JakobDegen): In this position, a binary would just fall back to + # the default link style. However, we have a little bit of additional + # information in the form of the preferred linkage that we can use to + # make a different decision. There's nothing technically wrong with + # that, but a comment explaining why we want to do it would be nice + doc_link_strategy = { + "any": LinkStrategy("shared"), + "shared": LinkStrategy("shared"), + "static": DEFAULT_STATIC_LINK_STRATEGY, + }[ctx.attrs.preferred_linkage] + + # The second thing we need is a lib output style of the regular, non-doctest + # version of this target that we want. Rustdoc does not handle this library + # being built in a "shared" way well, so this must be a static output style. + if ctx.attrs.doc_link_style: + doc_output_style = { + "shared": LibOutputStyle("pic_archive"), + "static": LibOutputStyle("archive"), + "static_pic": LibOutputStyle("pic_archive"), }[ctx.attrs.doc_link_style] else: - static_link_style = DEFAULT_STATIC_LINK_STYLE + doc_output_style = LibOutputStyle("pic_archive") + static_library_params = lang_style_param[(LinkageLang("rust"), doc_output_style)] - static_library_params = lang_style_param[(LinkageLang("rust"), static_link_style)] + # Among {rustdoc, doctests, macro expand}, doctests are the only one which + # cares about linkage. So whatever build params we picked for the doctests, + # reuse them for the other two as well default_roots = ["lib.rs"] rustdoc = generate_rustdoc( ctx = ctx, @@ -289,25 +269,25 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: document_private_items = False, ) + expand = rust_compile( + ctx = ctx, + compile_ctx = compile_ctx, + emit = Emit("expand"), + params = static_library_params, + default_roots = default_roots, + ) + # If doctests=True or False is set on the individual target, respect that. # Otherwise look at the global setting on the toolchain. doctests_enabled = \ (ctx.attrs.doctests if ctx.attrs.doctests != None else toolchain_info.doctests) and \ toolchain_info.rustc_target_triple == targets.exec_triple(ctx) - if ctx.attrs.doc_link_style: - doc_link_style = LinkStyle(ctx.attrs.doc_link_style) - else: - doc_link_style = { - "any": LinkStyle("shared"), - "shared": LinkStyle("shared"), - "static": DEFAULT_STATIC_LINK_STYLE, - }[ctx.attrs.preferred_linkage] rustdoc_test_params = build_params( rule = RuleType("binary"), proc_macro = ctx.attrs.proc_macro, - link_style = doc_link_style, - preferred_linkage = Linkage(ctx.attrs.preferred_linkage), + link_strategy = doc_link_strategy, + lib_output_style = None, lang = LinkageLang("rust"), linker_type = compile_ctx.cxx_toolchain_info.linker_info.type, target_os_type = ctx.attrs._target_os_type[OsLookup], @@ -315,23 +295,21 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: rustdoc_test = generate_rustdoc_test( ctx = ctx, compile_ctx = compile_ctx, - link_style = rustdoc_test_params.dep_link_style, - library = rust_param_artifact[static_library_params], + link_strategy = rustdoc_test_params.dep_link_strategy, + rlib = rust_param_artifact[static_library_params][0].output, params = rustdoc_test_params, default_roots = default_roots, ) - expand = rust_compile( + providers = [] + + link_infos = _link_infos( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("expand"), - params = static_library_params, - dep_link_style = DEFAULT_STATIC_LINK_STYLE, - default_roots = default_roots, + lang_style_param = lang_style_param, + param_artifact = native_param_artifact, ) - providers = [] - providers += _default_providers( lang_style_param = lang_style_param, param_artifact = rust_param_artifact, @@ -342,17 +320,21 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: expand = expand.output, sources = compile_ctx.symlinked_srcs, ) - providers += _rust_providers( + rust_link_info = _rust_providers( ctx = ctx, compile_ctx = compile_ctx, lang_style_param = lang_style_param, param_artifact = rust_param_artifact, + link_infos = link_infos, ) + providers.append(rust_link_info) providers += _native_providers( ctx = ctx, compile_ctx = compile_ctx, lang_style_param = lang_style_param, param_artifact = native_param_artifact, + link_infos = link_infos, + rust_link_info = rust_link_info, ) deps = [dep.dep for dep in resolve_deps(ctx, compile_ctx.dep_ctx)] @@ -370,7 +352,7 @@ def _build_params_for_styles( ctx: AnalysisContext, compile_ctx: CompileContext) -> ( dict[BuildParams, list[LinkageLang]], - dict[(LinkageLang, LinkStyle), BuildParams], + dict[(LinkageLang, LibOutputStyle), BuildParams], ): """ For a given rule, return two things: @@ -396,12 +378,12 @@ def _build_params_for_styles( if ctx.attrs.proc_macro and linkage_lang != LinkageLang("rust"): continue - for link_style in LinkStyle: + for lib_output_style in LibOutputStyle: params = build_params( rule = RuleType("library"), proc_macro = ctx.attrs.proc_macro, - link_style = link_style, - preferred_linkage = Linkage(ctx.attrs.preferred_linkage), + link_strategy = None, + lib_output_style = lib_output_style, lang = linkage_lang, linker_type = linker_type, target_os_type = target_os_type, @@ -409,7 +391,7 @@ def _build_params_for_styles( if params not in param_lang: param_lang[params] = [] param_lang[params] = param_lang[params] + [linkage_lang] - style_param[(linkage_lang, link_style)] = params + style_param[(linkage_lang, lib_output_style)] = params return (param_lang, style_param) @@ -424,8 +406,6 @@ def _build_library_artifacts( param_artifact = {} for params in params: - dep_link_style = params.dep_link_style - # Separate actions for each emit type # # In principle we don't really need metadata for C++-only artifacts, but I don't think it hurts @@ -434,7 +414,6 @@ def _build_library_artifacts( compile_ctx = compile_ctx, emits = [Emit("link"), Emit("metadata")], params = params, - dep_link_style = dep_link_style, default_roots = ["lib.rs"], ) @@ -445,20 +424,19 @@ def _build_library_artifacts( def _handle_rust_artifact( ctx: AnalysisContext, dep_ctx: DepCollectionContext, - params: BuildParams, + crate_type: CrateType, + link_strategy: LinkStrategy, link: RustcOutput, - meta: RustcOutput) -> RustLinkStyleInfo: + meta: RustcOutput) -> RustLinkStrategyInfo: """ Return the RustLinkInfo for a given set of artifacts. The main consideration is computing the right set of dependencies. """ - dep_link_style = params.dep_link_style - # If we're a crate where our consumers should care about transitive deps, # then compute them (specifically, not proc-macro). - if crate_type_transitive_deps(params.crate_type): - tdeps, tmetadeps, external_debug_info, tprocmacrodeps = _compute_transitive_deps(ctx, dep_ctx, dep_link_style) + if crate_type != CrateType("proc-macro"): + tdeps, tmetadeps, external_debug_info, tprocmacrodeps = _compute_transitive_deps(ctx, dep_ctx, link_strategy) else: tdeps, tmetadeps, external_debug_info, tprocmacrodeps = {}, {}, [], {} @@ -469,7 +447,7 @@ def _handle_rust_artifact( artifacts = filter(None, [link.dwo_output_directory]), children = external_debug_info, ) - return RustLinkStyleInfo( + return RustLinkStrategyInfo( rlib = link.output, transitive_deps = tdeps, rmeta = meta.output, @@ -480,7 +458,7 @@ def _handle_rust_artifact( ) else: # Proc macro deps are always the real thing - return RustLinkStyleInfo( + return RustLinkStrategyInfo( rlib = link.output, transitive_deps = tdeps, rmeta = link.output, @@ -491,10 +469,10 @@ def _handle_rust_artifact( ) def _default_providers( - lang_style_param: dict[(LinkageLang, LinkStyle), BuildParams], - param_artifact: dict[BuildParams, RustLinkStyleInfo], + lang_style_param: dict[(LinkageLang, LibOutputStyle), BuildParams], + param_artifact: dict[BuildParams, (RustcOutput, RustcOutput)], rustdoc: Artifact, - rustdoc_test: (cmd_args, dict[str, cmd_args]), + rustdoc_test: cmd_args, doctests_enabled: bool, check_artifacts: dict[str, Artifact], expand: Artifact, @@ -509,25 +487,31 @@ def _default_providers( for (k, v) in targets.items() } - # Add provider for default output, and for each link-style... - for link_style in LinkStyle: - link_style_info = param_artifact[lang_style_param[(LinkageLang("rust"), link_style)]] + # Add provider for default output, and for each lib output style... + # FIXME(JakobDegen): C++ rules only provide some of the output styles, + # determined by `get_output_styles_for_linkage` in `linking/link_info.bzl`. + # Do we want to do the same? + for output_style in LibOutputStyle: + link, _ = param_artifact[lang_style_param[(LinkageLang("rust"), output_style)]] nested_sub_targets = {} - if link_style_info.pdb: - nested_sub_targets[PDB_SUB_TARGET] = get_pdb_providers(pdb = link_style_info.pdb, binary = link_style_info.rlib) - sub_targets[link_style.value] = [DefaultInfo( - default_output = link_style_info.rlib, + if link.pdb: + nested_sub_targets[PDB_SUB_TARGET] = get_pdb_providers(pdb = link.pdb, binary = link.output) + + # FIXME(JakobDegen): Ideally we'd use the same + # `subtarget_for_output_style` as C++, but that uses `static-pic` + # instead of `static_pic`. Would be nice if that were consistent + name = legacy_output_style_to_link_style(output_style).value + sub_targets[name] = [DefaultInfo( + default_output = link.output, sub_targets = nested_sub_targets, )] providers = [] - (rustdoc_cmd, rustdoc_env) = rustdoc_test rustdoc_test_info = ExternalRunnerTestInfo( type = "rustdoc", - command = [rustdoc_cmd], + command = [rustdoc_test], run_from_project_root = True, - env = rustdoc_env, ) # Always let the user run doctests via `buck2 test :crate[doc]` @@ -544,101 +528,162 @@ def _default_providers( return providers +def _rust_link_providers( + ctx: AnalysisContext, + dep_ctx: DepCollectionContext, + cxx_toolchain: CxxToolchainInfo, + link_infos: dict[LibOutputStyle, LinkInfos], + preferred_linkage: Linkage) -> ( + MergedLinkInfo, + SharedLibraryInfo, + list[LinkableGraph], + list[Dependency], +): + # These are never accessed in the case of proc macros, so just return some dummy + # values + if ctx.attrs.proc_macro: + return ( + create_merged_link_info_for_propagation(ctx, []), + merge_shared_libraries(ctx.actions), + [], + [], + ) + + inherited_link_infos = inherited_merged_link_infos(ctx, dep_ctx) + inherited_shlibs = inherited_shared_libs(ctx, dep_ctx) + inherited_graphs = inherited_linkable_graphs(ctx, dep_ctx) + inherited_exported_deps = inherited_exported_link_deps(ctx, dep_ctx) + + if dep_ctx.advanced_unstable_linking: + # We have to produce a version of the providers that are defined in such + # a way that native rules looking at these providers will also pick up + # the `FORCE_RLIB` behavior. The general approach to that will be to + # claim that we have `preferred_linkage = "static"`. + # + # Note that all of this code is FORCE_RLIB specific. Disabling that + # setting requires replacing this with the "real" native providers + # + # As an optimization, we never bother reporting exported deps here. + # Whichever dependent uses the providers created here will take care of + # that for us. + merged_link_info = create_merged_link_info( + ctx, + cxx_toolchain.pic_behavior, + link_infos, + deps = inherited_link_infos, + preferred_linkage = Linkage("static"), + ) + shared_libs = merge_shared_libraries( + # We never actually have any shared libraries to add + ctx.actions, + deps = inherited_shlibs, + ) + + # The link graph representation is a little bit weird, since instead of + # just building up a graph via tsets, it uses a flat list of labeled + # nodes, each with a list of labels for dependency edges. The node that + # we create here cannot just use this target's label, since that would + # conflict with the node created for the native providers. As a result, + # we make up a fake subtarget to get a distinct label + new_label = ctx.label.configured_target().with_sub_target((ctx.label.sub_target or []) + ["fake_force_rlib_subtarget"]) + linkable_graph = create_linkable_graph( + ctx, + node = create_linkable_graph_node( + ctx, + linkable_node = create_linkable_node( + ctx = ctx, + preferred_linkage = Linkage("static"), + deps = inherited_graphs, + link_infos = link_infos, + # FIXME(JakobDegen): It should be ok to set this to `None`, + # but that breaks arc focus, and setting it to "" breaks + # somerge + default_soname = get_default_shared_library_name(cxx_toolchain.linker_info, ctx.label), + # Link groups have a heuristic in which they assume that a + # preferred_linkage = "static" library needs to be linked + # into every single link group, instead of just one. + # Applying that same heuristic to Rust seems right, but only + # if this target actually requested that. Opt ourselves out + # if it didn't. + ignore_force_static_follows_dependents = preferred_linkage != Linkage("static"), + ), + label = new_label, + ), + deps = inherited_graphs, + ) + + # We've already reported transitive deps on the inherited graphs, so for + # most purposes it would be fine to just have `linkable_graph` here. + # However, link groups do an analysis that relies on each symbol + # reference having a matching edge in the link graph, and so reexports + # and generics mean that we have to report a dependency on all + # transitive Rust deps and their immediate non-Rust deps + link_graphs = inherited_graphs + [linkable_graph] + else: + merged_link_info = create_merged_link_info_for_propagation(ctx, inherited_link_infos) + shared_libs = merge_shared_libraries( + ctx.actions, + deps = inherited_shlibs, + ) + link_graphs = inherited_graphs + return (merged_link_info, shared_libs, link_graphs, inherited_exported_deps) + def _rust_providers( ctx: AnalysisContext, compile_ctx: CompileContext, - lang_style_param: dict[(LinkageLang, LinkStyle), BuildParams], - param_artifact: dict[BuildParams, RustLinkStyleInfo]) -> list[Provider]: + lang_style_param: dict[(LinkageLang, LibOutputStyle), BuildParams], + param_artifact: dict[BuildParams, (RustcOutput, RustcOutput)], + link_infos: dict[LibOutputStyle, LinkInfos]) -> RustLinkInfo: """ Return the set of providers for Rust linkage. """ crate = attr_crate(ctx) - style_info = { - link_style: param_artifact[lang_style_param[(LinkageLang("rust"), link_style)]] - for link_style in LinkStyle - } + pic_behavior = compile_ctx.cxx_toolchain_info.pic_behavior + preferred_linkage = Linkage(ctx.attrs.preferred_linkage) - # Inherited link input and shared libraries. As in v1, this only includes - # non-Rust rules, found by walking through -- and ignoring -- Rust libraries - # to find non-Rust native linkables and libraries. - if not ctx.attrs.proc_macro: - inherited_link_deps = inherited_exported_link_deps(ctx, compile_ctx.dep_ctx) - inherited_link_infos = inherited_merged_link_infos(ctx, compile_ctx.dep_ctx) - inherited_shlibs = inherited_shared_libs(ctx, compile_ctx.dep_ctx) - else: - # proc-macros are just used by the compiler and shouldn't propagate - # their native deps to the link line of the target. - inherited_link_infos = [] - inherited_shlibs = [] - inherited_link_deps = [] + strategy_info = {} + for link_strategy in LinkStrategy: + params = lang_style_param[(LinkageLang("rust"), get_lib_output_style(link_strategy, preferred_linkage, pic_behavior))] + link, meta = param_artifact[params] + strategy_info[link_strategy] = _handle_rust_artifact(ctx, compile_ctx.dep_ctx, params.crate_type, link_strategy, link, meta) - providers = [] + merged_link_info, shared_libs, inherited_graphs, inherited_link_deps = _rust_link_providers(ctx, compile_ctx.dep_ctx, compile_ctx.cxx_toolchain_info, link_infos, Linkage(ctx.attrs.preferred_linkage)) # Create rust library provider. - providers.append(RustLinkInfo( + rust_link_info = RustLinkInfo( crate = crate, - styles = style_info, - merged_link_info = create_merged_link_info_for_propagation(ctx, inherited_link_infos), + strategies = strategy_info, + merged_link_info = merged_link_info, exported_link_deps = inherited_link_deps, - shared_libs = merge_shared_libraries( - ctx.actions, - deps = inherited_shlibs, - ), - )) + shared_libs = shared_libs, + linkable_graphs = inherited_graphs, + ) - return providers + return rust_link_info -def _native_providers( +def _link_infos( ctx: AnalysisContext, compile_ctx: CompileContext, - lang_style_param: dict[(LinkageLang, LinkStyle), BuildParams], - param_artifact: dict[BuildParams, RustcOutput]) -> list[Provider]: - """ - Return the set of providers needed to link Rust as a dependency for native - (ie C/C++) code, along with relevant dependencies. - """ + lang_style_param: dict[(LinkageLang, LibOutputStyle), BuildParams], + param_artifact: dict[BuildParams, RustcOutput]) -> dict[LibOutputStyle, LinkInfos]: + if ctx.attrs.proc_macro: + # Don't need any of this for proc macros + return {} - # If advanced_unstable_linking is set on the the rust toolchain, then build this artifact - # using the "native-unbundled" linkage language. See LinkageLang docs for more details advanced_unstable_linking = compile_ctx.toolchain_info.advanced_unstable_linking lang = LinkageLang("native-unbundled") if advanced_unstable_linking else LinkageLang("native") + linker_type = compile_ctx.cxx_toolchain_info.linker_info.type - inherited_link_deps = inherited_exported_link_deps(ctx, compile_ctx.dep_ctx) - inherited_link_infos = inherited_merged_link_infos(ctx, compile_ctx.dep_ctx) - inherited_shlibs = inherited_shared_libs(ctx, compile_ctx.dep_ctx) - linker_info = compile_ctx.cxx_toolchain_info.linker_info - linker_type = linker_info.type - - providers = [] - - if ctx.attrs.proc_macro: - # Proc-macros never have a native form - return providers - - # TODO(cjhopman): This seems to be conflating the link strategy with the lib output style. I tried going through - # lang_style_param/BuildParams and make it actually be based on LibOutputStyle, but it goes on to use that for defining - # how to consume dependencies and it's used for rust_binary like its own link strategy and it's unclear what's the - # correct behavior. For now, this preserves existing behavior without clarifying what concepts its actually - # operating on. - libraries = {} link_infos = {} - external_debug_infos = {} for output_style in LibOutputStyle: - legacy_link_style = legacy_output_style_to_link_style(output_style) - params = lang_style_param[(lang, legacy_link_style)] - lib = param_artifact[params] - libraries[output_style] = lib - - external_debug_info = inherited_external_debug_info( - ctx = ctx, - dep_ctx = compile_ctx.dep_ctx, - dwo_output_directory = lib.dwo_output_directory, - dep_link_style = params.dep_link_style, + lib = param_artifact[lang_style_param[(lang, output_style)]] + external_debug_info = make_artifact_tset( + actions = ctx.actions, + label = ctx.label, + artifacts = filter(None, [lib.dwo_output_directory]), + children = lib.extra_external_debug_info, ) - external_debug_infos[output_style] = external_debug_info - - # DO NOT COMMIT: verify this change if output_style == LibOutputStyle("shared_lib"): link_infos[output_style] = LinkInfos( default = LinkInfo( @@ -646,16 +691,8 @@ def _native_providers( external_debug_info = external_debug_info, ), stripped = LinkInfo( - linkables = [ArchiveLinkable( - archive = Archive( - artifact = strip_debug_info( - ctx, - paths.join(output_style.value, lib.output.short_path), - lib.output, - ), - ), - linker_type = linker_type, - )], + linkables = [SharedLibLinkable(lib = lib.stripped_output)], + external_debug_info = external_debug_info, ), ) else: @@ -667,7 +704,58 @@ def _native_providers( )], external_debug_info = external_debug_info, ), + stripped = LinkInfo( + linkables = [ArchiveLinkable( + archive = Archive(artifact = lib.stripped_output), + linker_type = linker_type, + )], + ), ) + return link_infos + +def _native_providers( + ctx: AnalysisContext, + compile_ctx: CompileContext, + lang_style_param: dict[(LinkageLang, LibOutputStyle), BuildParams], + param_artifact: dict[BuildParams, RustcOutput], + link_infos: dict[LibOutputStyle, LinkInfos], + rust_link_info: RustLinkInfo) -> list[Provider]: + """ + Return the set of providers needed to link Rust as a dependency for native + (ie C/C++) code, along with relevant dependencies. + """ + + if ctx.attrs.proc_macro: + # Proc-macros never have a native form + return [] + + # If advanced_unstable_linking is set on the the rust toolchain, then build this artifact + # using the "native-unbundled" linkage language. See LinkageLang docs for more details + advanced_unstable_linking = compile_ctx.toolchain_info.advanced_unstable_linking + lang = LinkageLang("native-unbundled") if advanced_unstable_linking else LinkageLang("native") + + if advanced_unstable_linking: + # The rust link providers already contain the linkables for the `archive` and `pic_archive` + # cases + link_infos = { + LibOutputStyle("shared_lib"): link_infos[LibOutputStyle("shared_lib")], + LibOutputStyle("archive"): LinkInfos(default = LinkInfo()), + LibOutputStyle("pic_archive"): LinkInfos(default = LinkInfo()), + } + + # We collected transitive deps in the Rust link providers + inherited_link_infos = [rust_link_info.merged_link_info] + inherited_shlibs = [rust_link_info.shared_libs] + inherited_link_graphs = rust_link_info.linkable_graphs + inherited_exported_deps = rust_link_info.exported_link_deps + + linker_info = compile_ctx.cxx_toolchain_info.linker_info + linker_type = linker_info.type + + providers = [] + + shared_lib_params = lang_style_param[(lang, LibOutputStyle("shared_lib"))] + shared_lib_output = param_artifact[shared_lib_params].output preferred_linkage = Linkage(ctx.attrs.preferred_linkage) @@ -676,14 +764,14 @@ def _native_providers( ctx, compile_ctx.cxx_toolchain_info.pic_behavior, link_infos, - exported_deps = inherited_link_infos, + deps = inherited_link_infos, + exported_deps = filter(None, [d.get(MergedLinkInfo) for d in inherited_exported_deps]), preferred_linkage = preferred_linkage, )) solibs = {} # Add the shared library to the list of shared libs. - linker_info = compile_ctx.cxx_toolchain_info.linker_info shlib_name = get_default_shared_library_name(linker_info, ctx.label) # Only add a shared library if we generated one. @@ -692,9 +780,9 @@ def _native_providers( # to remove the SharedLibraries provider, maybe just wait for that to resolve this. if get_lib_output_style(LinkStrategy("shared"), preferred_linkage, compile_ctx.cxx_toolchain_info.pic_behavior) == LibOutputStyle("shared_lib"): solibs[shlib_name] = LinkedObject( - output = libraries[LibOutputStyle("shared_lib")].output, - unstripped_output = libraries[LibOutputStyle("shared_lib")].output, - external_debug_info = external_debug_infos[LibOutputStyle("shared_lib")], + output = shared_lib_output, + unstripped_output = shared_lib_output, + external_debug_info = link_infos[LibOutputStyle("shared_lib")].default.external_debug_info, ) # Native shared library provider. @@ -711,15 +799,15 @@ def _native_providers( default = LinkInfo( linkables = [ArchiveLinkable( archive = Archive( - artifact = libraries[LibOutputStyle("shared_lib")].output, + artifact = shared_lib_output, ), linker_type = linker_type, link_whole = True, )], - external_debug_info = external_debug_infos[LibOutputStyle("pic_archive")], + external_debug_info = link_infos[LibOutputStyle("pic_archive")].default.external_debug_info, ), ), - deps = inherited_link_deps, + deps = inherited_link_graphs, ) providers.append(linkable_root) @@ -734,18 +822,22 @@ def _native_providers( linkable_node = create_linkable_node( ctx = ctx, preferred_linkage = preferred_linkage, - exported_deps = inherited_link_deps, + deps = inherited_link_graphs, + exported_deps = inherited_exported_deps, link_infos = link_infos, shared_libs = solibs, default_soname = shlib_name, ), ), - deps = inherited_link_deps, + deps = inherited_link_graphs + inherited_exported_deps, ) providers.append(linkable_graph) - providers.append(merge_link_group_lib_info(deps = inherited_link_deps)) + # We never need to add anything to this provider because Rust libraries + # cannot act as link group libs, especially given that they only support + # auto link groups anyway + providers.append(merge_link_group_lib_info(children = inherited_link_group_lib_infos(ctx, compile_ctx.dep_ctx))) return providers @@ -753,7 +845,7 @@ def _native_providers( def _compute_transitive_deps( ctx: AnalysisContext, dep_ctx: DepCollectionContext, - dep_link_style: LinkStyle) -> ( + dep_link_strategy: LinkStrategy) -> ( dict[Artifact, CrateName], dict[Artifact, CrateName], list[ArtifactTSet], @@ -770,16 +862,16 @@ def _compute_transitive_deps( # We don't want to propagate proc macros directly, and they have no transitive deps continue - style = style_info(dep.info, dep_link_style) - transitive_deps[style.rlib] = dep.info.crate - transitive_deps.update(style.transitive_deps) + strategy = strategy_info(dep.info, dep_link_strategy) + transitive_deps[strategy.rlib] = dep.info.crate + transitive_deps.update(strategy.transitive_deps) - transitive_rmeta_deps[style.rmeta] = dep.info.crate - transitive_rmeta_deps.update(style.transitive_rmeta_deps) + transitive_rmeta_deps[strategy.rmeta] = dep.info.crate + transitive_rmeta_deps.update(strategy.transitive_rmeta_deps) - external_debug_info.append(style.external_debug_info) + external_debug_info.append(strategy.external_debug_info) - transitive_proc_macro_deps.update(style.transitive_proc_macro_deps) + transitive_proc_macro_deps.update(strategy.transitive_proc_macro_deps) return transitive_deps, transitive_rmeta_deps, external_debug_info, transitive_proc_macro_deps diff --git a/prelude/rust/rust_toolchain.bzl b/prelude/rust/rust_toolchain.bzl index b0a66a97868a4..facf9a7688bf0 100644 --- a/prelude/rust/rust_toolchain.bzl +++ b/prelude/rust/rust_toolchain.bzl @@ -91,7 +91,6 @@ rust_toolchain_attrs = { # linking types in signatures to their definition in another crate. "extern_html_root_url_prefix": provider_field(str | None, default = None), # Utilities used for building flagfiles containing dynamic crate names - "concat_tool": provider_field(RunInfo | None, default = None), "transitive_dependency_symlinks_tool": provider_field(RunInfo | None, default = None), # Setting this enables additional behaviors that improves linking at the # cost of using unstable implementation details of rustc. At the moment, diff --git a/prelude/rust/tools/BUCK.v2 b/prelude/rust/tools/BUCK.v2 index c40ac525c2e5a..f3f1bada5ec32 100644 --- a/prelude/rust/tools/BUCK.v2 +++ b/prelude/rust/tools/BUCK.v2 @@ -25,12 +25,6 @@ prelude.python_bootstrap_binary( visibility = ["PUBLIC"], ) -prelude.python_bootstrap_binary( - name = "concat", - main = "concat.py", - visibility = ["PUBLIC"], -) - prelude.python_bootstrap_binary( name = "transitive_dependency_symlinks", main = "transitive_dependency_symlinks.py", diff --git a/prelude/rust/tools/attrs.bzl b/prelude/rust/tools/attrs.bzl index 7d4231e8f13bc..f5fb89307034e 100644 --- a/prelude/rust/tools/attrs.bzl +++ b/prelude/rust/tools/attrs.bzl @@ -12,7 +12,6 @@ def _internal_tool(default: str) -> Attr: # configurable attributes there. This list of internal tools is distracting and # expected to grow. internal_tool_attrs = { - "concat_tool": _internal_tool("prelude//rust/tools:concat"), "failure_filter_action": _internal_tool("prelude//rust/tools:failure_filter_action"), "rustc_action": _internal_tool("prelude//rust/tools:rustc_action"), "rustdoc_test_with_resources": _internal_tool("prelude//rust/tools:rustdoc_test_with_resources"), diff --git a/prelude/rust/tools/concat.py b/prelude/rust/tools/concat.py deleted file mode 100755 index 6dfb8723fdb57..0000000000000 --- a/prelude/rust/tools/concat.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under both the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree and the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. - -# A tool to concatenate strings, some of which may be from @files. ¯\_(ツ)_/¯ -# -# Rustc's command line requires dependencies to be provided as: -# -# --extern cratename=path/to/libcratename.rlib -# -# In Buck, sometimes the cratename is computed at build time, for example -# extracted from a Thrift file. Rustc's "@" support isn't sufficient for this -# because the following doesn't make sense: -# -# --extern @filecontainingcrate=path/to/libcratename.rlib -# -# and the cratename isn't able to be its own argument: -# -# --extern @filecontainingcrate =path/to/libcratename.rlib -# -# Instead we use Python to make a single file containing the dynamic cratename -# and the rlib filepath concatenated together. -# -# concat.py --output $TMP -- @filecontainingcrate = path/to/libcratename.rlib -# -# then: -# -# --extern @$TMP -# - -import argparse -from typing import IO, List, NamedTuple - - -class Args(NamedTuple): - output: IO[str] - strings: List[str] - - -def main(): - parser = argparse.ArgumentParser(fromfile_prefix_chars="@") - parser.add_argument("--output", type=argparse.FileType("w")) - parser.add_argument("strings", nargs="*", type=str) - args = Args(**vars(parser.parse_args())) - - args.output.write("".join(args.strings)) - - -if __name__ == "__main__": - main() diff --git a/prelude/rust/tools/rustc_action.py b/prelude/rust/tools/rustc_action.py index a4097076bf589..b6f9984416665 100755 --- a/prelude/rust/tools/rustc_action.py +++ b/prelude/rust/tools/rustc_action.py @@ -58,7 +58,6 @@ class Args(NamedTuple): buck_target: Optional[str] failure_filter: Optional[IO[bytes]] required_output: Optional[List[Tuple[str, str]]] - only_artifact: Optional[str] rustc: List[str] @@ -119,12 +118,6 @@ def arg_parse() -> Args: help="Required output path we expect rustc to generate " "(and filled with a placeholder on a filtered failure)", ) - parser.add_argument( - "--only-artifact", - metavar="TYPE", - help="Terminate rustc after requested artifact type (metadata, link, etc) has been emitted. " - "(Assumes compiler is invoked with --error-format=json --json=artifacts)", - ) parser.add_argument( "rustc", nargs=argparse.REMAINDER, @@ -135,18 +128,35 @@ def arg_parse() -> Args: return Args(**vars(parser.parse_args())) +def arg_eval(arg: str) -> str: + """ + Expand an argument such as --extern=$(cat buck-out/v2/gen/foo.txt)=buck-out/dev/gen/libfoo.rlib + """ + expanded = "" + + while True: + begin = arg.find("$(cat ") + if begin == -1: + return expanded + arg + expanded += arg[:begin] + begin += len("$(cat ") + path, rest = arg[begin:].split(")", maxsplit=1) + with open(path, encoding="utf-8") as f: + expanded += f.read().strip() + arg = rest + + async def handle_output( # noqa: C901 proc: asyncio.subprocess.Process, args: Args, crate_map: Dict[str, str], -) -> Tuple[bool, bool]: +) -> bool: got_error_diag = False - shutdown = False proc_stderr = proc.stderr assert proc_stderr is not None - while not shutdown: + while True: line = await proc_stderr.readline() if line is None or line == b"": @@ -161,12 +171,7 @@ async def handle_output( # noqa: C901 if DEBUG: print(f"diag={repr(diag)}", end="\n") - # We have to sniff the shape of diag record based on what fields it has set. - if "artifact" in diag and "emit" in diag: - if diag["emit"] == args.only_artifact: - shutdown = True - continue - elif "unused_extern_names" in diag: + if "unused_extern_names" in diag: unused_names = diag["unused_extern_names"] # Empty unused_extern_names is just noise. @@ -219,7 +224,7 @@ async def handle_output( # noqa: C901 if args.diag_txt: args.diag_txt.close() - return (got_error_diag, shutdown) + return got_error_diag async def main() -> int: @@ -274,7 +279,7 @@ async def main() -> int: print(f"args {repr(args)} env {env} crate_map {crate_map}", end="\n") rustc_cmd = args.rustc[:1] - rustc_args = args.rustc[1:] + rustc_args = [arg_eval(arg) for arg in args.rustc[1:]] if args.remap_cwd_prefix is not None: rustc_args.append( @@ -304,24 +309,12 @@ async def main() -> int: stderr=subprocess.PIPE, limit=1_000_000, ) - (got_error_diag, shutdown) = await handle_output(proc, args, crate_map) - - if shutdown: - # We got what we want so shut down early - try: - proc.terminate() - except ProcessLookupError: - # The process already terminated on its own. - pass - await proc.wait() - res = 0 - else: - res = await proc.wait() + got_error_diag = await handle_output(proc, args, crate_map) + res = await proc.wait() if DEBUG: print( f"res={repr(res)} " - f"shutdown={shutdown} " f"got_error_diag={got_error_diag} " f"args.failure_filter {args.failure_filter}", end="\n", @@ -333,7 +326,7 @@ async def main() -> int: # Check for death by signal - this is always considered a failure if res < 0: - cmdline = " ".join(shlex.quote(arg) for arg in args.rustc) + cmdline = " ".join(shlex.quote(arg) for arg in rustc_cmd + rustc_args) eprint(f"Command exited with signal {-res}: command line: {cmdline}") elif args.failure_filter: # If failure filtering is enabled, then getting an error diagnostic is also @@ -371,4 +364,4 @@ async def main() -> int: # There is a bug with asyncio.run() on Windows: # https://bugs.python.org/issue39232 -sys.exit(asyncio.get_event_loop().run_until_complete(main())) +sys.exit(asyncio.new_event_loop().run_until_complete(main())) diff --git a/prelude/rust/tools/transitive_dependency_symlinks.py b/prelude/rust/tools/transitive_dependency_symlinks.py index 247d683fc3bb9..77959079b63ec 100755 --- a/prelude/rust/tools/transitive_dependency_symlinks.py +++ b/prelude/rust/tools/transitive_dependency_symlinks.py @@ -29,22 +29,31 @@ # # transitive_dependency_symlinks.py \ # --out-dir path/to/out \ -# --artifact path/to/cratename ../../libprovisional.rlib \ -# --artifact ... +# --artifacts path/to/artifacts.json # -# The tool reads the crate name from the file at "path/to/out". Suppose it's +# The input file artifact.json is an array of pairs, each an rlib and a file +# containing a crate name for it. +# +# [ +# ["../../libprovisional.rlib", "path/to/cratename"], +# ... +# ] +# +# The tool reads the crate name from the file at "path/to/cratename". Suppose it's # "thriftgenerated". It symlinks the given artifact as "0/libthriftgenerated.rlib" # within the specified output directory. In the event of collisions, there might # be multiple dirs created, just as we do for analysis-time named crates. import argparse +import json +import os from pathlib import Path -from typing import List, NamedTuple, Tuple +from typing import IO, NamedTuple class Args(NamedTuple): out_dir: Path - artifact: List[Tuple[Path, Path]] + artifacts: IO[str] def main(): @@ -55,11 +64,8 @@ def main(): required=True, ) parser.add_argument( - "--artifact", - action="append", - nargs=2, - type=Path, - metavar=("CRATENAME", "ARTIFACT"), + "--artifacts", + type=argparse.FileType(), required=True, ) args = Args(**vars(parser.parse_args())) @@ -69,9 +75,9 @@ def main(): # Add as many -Ldependency dirs as we need to avoid name conflicts deps_dirs = [{}] - for crate_name, artifact in args.artifact: - crate_name = crate_name.read_text().strip() - original_filename = artifact.name + for artifact, crate_name in json.load(args.artifacts): + crate_name = Path(crate_name).read_text().strip() + original_filename = os.path.basename(artifact) new_filename = "lib{}-{}".format( crate_name, original_filename.rsplit("-", 1)[1], diff --git a/prelude/tests/re_utils.bzl b/prelude/tests/re_utils.bzl index eacb448d35cd7..514396604ff62 100644 --- a/prelude/tests/re_utils.bzl +++ b/prelude/tests/re_utils.bzl @@ -7,7 +7,7 @@ load("@prelude//:build_mode.bzl", "BuildModeInfo") load("@prelude//tests:remote_test_execution_toolchain.bzl", "RemoteTestExecutionToolchainInfo") -load("@prelude//utils:utils.bzl", "expect_non_none") +load("@prelude//utils:expect.bzl", "expect_non_none") def _get_re_arg(ctx: AnalysisContext): if not hasattr(ctx.attrs, "remote_execution"): @@ -44,6 +44,7 @@ def get_re_executors_from_props(ctx: AnalysisContext) -> ([CommandExecutorConfig use_case = re_props_copy.pop("use_case") listing_capabilities = re_props_copy.pop("listing_capabilities", None) remote_cache_enabled = re_props_copy.pop("remote_cache_enabled", None) + re_dependencies = re_props_copy.pop("dependencies", []) if re_props_copy: unexpected_props = ", ".join(re_props_copy.keys()) fail("found unexpected re props: " + unexpected_props) @@ -60,6 +61,7 @@ def get_re_executors_from_props(ctx: AnalysisContext) -> ([CommandExecutorConfig remote_execution_use_case = use_case or "tpx-default", remote_cache_enabled = remote_cache_enabled, remote_execution_action_key = remote_execution_action_key, + remote_execution_dependencies = re_dependencies, ) listing_executor = default_executor if listing_capabilities: diff --git a/prelude/toolchains/haskell.bzl b/prelude/toolchains/haskell.bzl index fd1384616fced..c3e99c382af4e 100644 --- a/prelude/toolchains/haskell.bzl +++ b/prelude/toolchains/haskell.bzl @@ -19,7 +19,7 @@ def _system_haskell_toolchain(_ctx: AnalysisContext) -> list[Provider]: linker_flags = [], ), HaskellPlatformInfo( - name = "x86_64", + name = host_info().arch, ), ] diff --git a/prelude/toolchains/rust.bzl b/prelude/toolchains/rust.bzl index 3b4972daaea58..d018eddedc006 100644 --- a/prelude/toolchains/rust.bzl +++ b/prelude/toolchains/rust.bzl @@ -42,7 +42,6 @@ def _system_rust_toolchain_impl(ctx): clippy_driver = RunInfo(args = ["clippy-driver"]), clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None, compiler = RunInfo(args = ["rustc"]), - concat_tool = ctx.attrs.concat_tool[RunInfo], default_edition = ctx.attrs.default_edition, panic_runtime = PanicRuntime("unwind"), deny_lints = ctx.attrs.deny_lints, @@ -74,7 +73,7 @@ system_rust_toolchain = rule( "deny_lints": attrs.list(attrs.string(), default = []), "doctests": attrs.bool(default = False), "extern_html_root_url_prefix": attrs.option(attrs.string(), default = None), - "pipelined": attrs.bool(default = False), + "pipelined": attrs.bool(default = True), "report_unused_deps": attrs.bool(default = False), "rustc_binary_flags": attrs.list(attrs.string(), default = []), "rustc_check_flags": attrs.list(attrs.string(), default = []), diff --git a/prelude/utils/expect.bzl b/prelude/utils/expect.bzl index ed41d76636158..7635ac6f5efa3 100644 --- a/prelude/utils/expect.bzl +++ b/prelude/utils/expect.bzl @@ -32,9 +32,17 @@ def expect(condition: typing.Any, message: str = "condition not expected", *form format_args: optional arguments to format the error message with """ if not condition: - formatted_message = message.format(format_args) + formatted_message = message.format(*format_args) fail(formatted_message) +def expect_non_none(val, msg: str = "unexpected none", *fmt_args, **fmt_kwargs): + """ + Require the given value not be `None`. + """ + if val == None: + fail(msg.format(*fmt_args, **fmt_kwargs)) + return val + def expect_type(name: str, check: typing.Callable, desc: str, val: typing.Any): """Fails if check(val) if not truthy. name, desc are used for the error message. diff --git a/prelude/utils/utils.bzl b/prelude/utils/utils.bzl index 1cf1ea7701ba9..cecc99d363d51 100644 --- a/prelude/utils/utils.bzl +++ b/prelude/utils/utils.bzl @@ -7,6 +7,8 @@ # General utilities shared between multiple rules. +load("@prelude//utils:expect.bzl", "expect") + def value_or(x: [None, typing.Any], default: typing.Any) -> typing.Any: return default if x == None else x @@ -18,20 +20,6 @@ def flatten(xss: list[list[typing.Any]]) -> list[typing.Any]: def flatten_dict(xss: list[dict[typing.Any, typing.Any]]) -> dict[typing.Any, typing.Any]: return {k: v for xs in xss for k, v in xs.items()} -# Fail if given condition is not met. -def expect(x: bool, msg: str = "condition not expected", *fmt): - if not x: - fmt_msg = msg.format(*fmt) - fail(fmt_msg) - -def expect_non_none(val, msg: str = "unexpected none", *fmt_args, **fmt_kwargs): - """ - Require the given value not be `None`. - """ - if val == None: - fail(msg.format(*fmt_args, **fmt_kwargs)) - return val - def from_named_set(srcs: [dict[str, Artifact | Dependency], list[Artifact | Dependency]]) -> dict[str, Artifact | Dependency]: """ Normalize parameters of optionally named sources to a dictionary mapping diff --git a/prelude/zip_file/tools/unzip.py b/prelude/zip_file/tools/unzip.py index e571c3987fa58..3ec289156e4eb 100644 --- a/prelude/zip_file/tools/unzip.py +++ b/prelude/zip_file/tools/unzip.py @@ -28,6 +28,11 @@ def do_unzip(archive, output_dir): # That way we don't need to pass `target_is_directory` argument to `os.symlink` function. for info in (i for i in z.infolist() if not _is_symlink(i)): z.extract(info, path=output_dir) + if _is_executable(info): + os.chmod( + os.path.join(output_dir, info.filename), + _file_attributes(info) | stat.S_IXUSR, + ) for info in (i for i in z.infolist() if _is_symlink(i)): symlink_path = os.path.join(output_dir, info.filename) symlink_dst = z.read(info).decode("utf-8") @@ -54,6 +59,10 @@ def _is_symlink(zip_info): return stat.S_ISLNK(_file_attributes(zip_info)) +def _is_executable(zip_info): + return stat.S_IMODE(_file_attributes(zip_info)) & stat.S_IXUSR + + def main(): args = _parse_args() print("Source zip is: {}".format(args.src), file=sys.stderr) diff --git a/remote_execution/oss/re_grpc/src/lib.rs b/remote_execution/oss/re_grpc/src/lib.rs index a41e71778ad17..ea695ca620537 100644 --- a/remote_execution/oss/re_grpc/src/lib.rs +++ b/remote_execution/oss/re_grpc/src/lib.rs @@ -15,7 +15,6 @@ mod metadata; mod request; mod response; pub use client::*; -pub use digest::*; pub use error::*; pub use grpc::*; pub use metadata::*; diff --git a/remote_execution/oss/re_grpc/src/metadata.rs b/remote_execution/oss/re_grpc/src/metadata.rs index b46de83be4c27..2ee092ac25366 100644 --- a/remote_execution/oss/re_grpc/src/metadata.rs +++ b/remote_execution/oss/re_grpc/src/metadata.rs @@ -27,6 +27,7 @@ pub struct HostResourceRequirements { #[derive(Clone, Default)] pub struct BuckInfo { pub build_id: String, + pub version: String, pub _dot_dot: (), } diff --git a/rust-toolchain b/rust-toolchain index 0af5f9624e7b3..a69e38c288385 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -4,8 +4,11 @@ # * Update the `rustc_version` directive (read by `app/buck2_core/build.rs`). # * Update `HACKING.md` (two instances). # * Update `docs/getting_started.md` (two instances). -# * Update `../common/ocaml/interop/oss/README-BUCK.md` (two instances), `../common/ocaml/interop/.circleci/config.yml` (one instance) and `../common/ocaml/interop/facebook/ci_oss.sh` (one instance). +# * Update `../common/rust/tools/reindeer/rust-toolchain` (one instance) +# * Update `../common/ocaml/interop/rust-toolchain` (one instance) +# * NOTE: You may have to change this file in a follow up commit as ocamlrep +# has a dependency on buck2 git trunk. -# @rustc_version: rustc 1.74.0-nightly (ca62d2c44 2023-09-30) -channel = "nightly-2023-10-01" +# @rustc_version: rustc 1.75.0-nightly (0f44eb32f 2023-11-09) +channel = "nightly-2023-11-10" components = ["llvm-tools-preview","rustc-dev","rust-src"] diff --git a/shed/README.md b/shed/README.md index 3d4fa4912e0a6..5f80b804c4449 100644 --- a/shed/README.md +++ b/shed/README.md @@ -1,6 +1,8 @@ # Shed Code which is: -* used by Buck -* generic, knows nothing of Buck -* we would rather not have written and would like to get into a different package + +- used by Buck +- generic, knows nothing of Buck +- we would rather not have written and would like to get into a different + package diff --git a/shim/README.md b/shim/README.md index 6b3b272221541..7a83226e02ac5 100644 --- a/shim/README.md +++ b/shim/README.md @@ -1,3 +1,4 @@ # Open Source Shim -These files are a shim that allow us to build Buck2 with Buck2 outside Meta in the open source world. +These files are a shim that allow us to build Buck2 with Buck2 outside Meta in +the open source world. diff --git a/shim/third-party/rust/Cargo.toml b/shim/third-party/rust/Cargo.toml index 5021d0f0f79ad..8e634dbbd7364 100644 --- a/shim/third-party/rust/Cargo.toml +++ b/shim/third-party/rust/Cargo.toml @@ -34,7 +34,7 @@ assert_matches = "1.5" async-compression = { version = "0.4.1", features = ["tokio", "gzip", "zstd"] } async-condvar-fair = { version = "0.2.2", features = ["parking_lot_0_11", "tokio"] } async-recursion = "1.0" -async-scoped = { version = "0.7.1", features = ["use-tokio"] } +async-scoped = { version = "0.8", features = ["use-tokio"] } async-trait = "0.1.24" atomic = "0.5.1" backtrace = "0.3.51" @@ -59,7 +59,7 @@ crossbeam-epoch = "0.9.7" crossterm = "0.27" csv = "1.1" ctor = "0.1.16" -dashmap = "4.0.2" +dashmap = "5.5.3" debugserver-types = "0.5.0" derivative = "2.2" derive_more = "0.99.3" @@ -79,9 +79,10 @@ fnv = "1.0.7" fs4 = { version = "0.6", features = ["sync"] } futures = { version = "0.3.28", features = ["async-await", "compat"] } futures-intrusive = "0.4" +fxhash = "0.2.1" glob = "0.3.0" globset = "0.4.10" -hashbrown = { version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.3", features = ["raw"] } hex = "0.4.3" higher-order-closure = "0.0.5" hostname = "0.3.1" @@ -132,6 +133,7 @@ once_cell = "1.8" os_str_bytes = { version = "6.6.0", features = ["conversions"] } parking_lot = { version = "0.11.2", features = ["send_guard"] } paste = "1.0" +pathdiff = "0.2" perf-event = "0.4" perf-event-open-sys = "4.0" pin-project = "0.4.29" @@ -159,11 +161,11 @@ rustls-pemfile = { package = "rustls-pemfile", version = "1.0.0" } rustyline = "11.0" scopeguard = "1.0.0" sequence_trie = "0.3.6" -serde = { version = "1.0.173", features = ["derive"] } +serde = { version = "1.0.173", features = ["derive", "rc"] } serde_json = "1.0.48" sha1 = "0.10" sha2 = "0.10" -shlex = "1.0" +shlex = "1.3" siphasher = "0.3.3" slab = "0.4.7" slog = "2.7.0" @@ -197,7 +199,7 @@ tower-layer = "0.3.1" tower-service = "0.3.2" tracing = "0.1.22" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -triomphe = "0.1.8" +triomphe = "0.1.11" trybuild = "1.0.56" twox-hash = "1.6.1" unicode-segmentation = "1.7" diff --git a/starlark-rust/README.md b/starlark-rust/README.md index 1c53f58201c52..4d6a6ad36694d 100644 --- a/starlark-rust/README.md +++ b/starlark-rust/README.md @@ -6,11 +6,25 @@ [![docs.rs availability](https://img.shields.io/docsrs/starlark?label=docs.rs)](https://docs.rs/starlark/) [![Build status](https://img.shields.io/github/actions/workflow/status/facebookexperimental/starlark-rust/ci.yml?branch=main)](https://github.com/facebookexperimental/starlark-rust/actions) -There are several copies of this repo on GitHub, [facebookexperimental/starlark-rust](https://github.com/facebookexperimental/starlark-rust) is the canonical one. - -This project provides a Rust implementation of the [Starlark language](https://github.com/bazelbuild/starlark/blob/master/spec.md). Starlark (formerly codenamed Skylark) is a deterministic language inspired by Python3, used for configuration in the build systems [Bazel](https://bazel.build), [Buck](https://buck.build) and [Buck2](https://buck2.build), of which Buck2 depends on this library. This project was originally developed [in this repo](https://github.com/google/starlark-rust), which contains a more extensive history. - -There are at least three implementations of Starlark, [one in Java](https://github.com/bazelbuild/starlark), [one in Go](https://github.com/google/starlark-go), and this one in Rust. We mostly follow the Starlark standard. If you are interested in trying out Rust Starlark, you can clone this repo and run: +There are several copies of this repo on GitHub, +[facebookexperimental/starlark-rust](https://github.com/facebookexperimental/starlark-rust) +is the canonical one. + +This project provides a Rust implementation of the +[Starlark language](https://github.com/bazelbuild/starlark/blob/master/spec.md). +Starlark (formerly codenamed Skylark) is a deterministic language inspired by +Python3, used for configuration in the build systems +[Bazel](https://bazel.build), [Buck](https://buck.build) and +[Buck2](https://buck2.build), of which Buck2 depends on this library. This +project was originally developed +[in this repo](https://github.com/google/starlark-rust), which contains a more +extensive history. + +There are at least three implementations of Starlark, +[one in Java](https://github.com/bazelbuild/starlark), +[one in Go](https://github.com/google/starlark-go), and this one in Rust. We +mostly follow the Starlark standard. If you are interested in trying out Rust +Starlark, you can clone this repo and run: ```shell $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh @@ -19,58 +33,110 @@ $> 1+2 3 ``` -This project was started by [Damien Martin-Guillerez](https://github.com/damienmg). Version 0.4.0 of this library changed ownership [from Google](https://github.com/google/starlark-rust) to Facebook. +This project was started by +[Damien Martin-Guillerez](https://github.com/damienmg). Version 0.4.0 of this +library changed ownership [from Google](https://github.com/google/starlark-rust) +to Facebook. ## Learn More -Read [this blog post](https://developers.facebook.com/blog/post/2021/04/08/rust-starlark-library/) for an overview of the library, the reasons behind Starlark, and how it might fit in to your project. There is also a [2 minute introductory video](https://www.youtube.com/watch?v=3kHER3KIPj4). +Read +[this blog post](https://developers.facebook.com/blog/post/2021/04/08/rust-starlark-library/) +for an overview of the library, the reasons behind Starlark, and how it might +fit in to your project. There is also a +[2 minute introductory video](https://www.youtube.com/watch?v=3kHER3KIPj4). ## Features This project features: -* Easy interoperability between Rust types and Starlark. -* Rust-friendly types, so frozen values are `Send`/`Sync`, while non-frozen values aren't. -* [Garbage collected](docs/gc.md) values allocated on [a heap](docs/heaps.md). -* Optional runtime-checked [types](docs/types.md). -* A linter, to detect code issues in Starlark. -* IDE integration in the form of [LSP](https://microsoft.github.io/language-server-protocol/). - +- Easy interoperability between Rust types and Starlark. +- Rust-friendly types, so frozen values are `Send`/`Sync`, while non-frozen + values aren't. +- [Garbage collected](docs/gc.md) values allocated on [a heap](docs/heaps.md). +- Optional runtime-checked [types](docs/types.md). +- A linter, to detect code issues in Starlark. +- IDE integration in the form of + [LSP](https://microsoft.github.io/language-server-protocol/). +- Extensive testing, including + [fuzz testing](https://github.com/google/oss-fuzz/tree/master/projects/starlark-rust). +- [DAP](https://microsoft.github.io/debug-adapter-protocol/) support. This project also has three non-goals: -* We do _not_ aim for API stability between releases, preferring to iterate quickly and refine the API as much as possible. But we do [follow SemVer](https://doc.rust-lang.org/cargo/reference/semver.html). -* We do _not_ aim for minimal dependencies, preferring to keep one package with lots of power. But if some dependencies prove tricky, we might add feature flags. +- We do _not_ aim for API stability between releases, preferring to iterate + quickly and refine the API as much as possible. But we do + [follow SemVer](https://doc.rust-lang.org/cargo/reference/semver.html). +- We do _not_ aim for minimal dependencies, preferring to keep one package with + lots of power. But if some dependencies prove tricky, we might add feature + flags. ## Components -There are three components: - -* `starlark_derive`, a proc-macro crate that defines the necessary macros for Starlark. This library is a dependency of `starlark` the library, which reexports all the relevant pieces, and should not be used directly. -* `starlark` the library, a library that defines the parser, evaluator and standard library. Projects wishing to embed Starlark in their environment (with additional types, library functions and features) will make use of this library. -* `starlark` the binary, which provides interactive evaluation, IDE features and linter, exposed through a command line. Useful if you want to use vanilla Starlark (but if you do, consider Python3 instead) or as a test-bed for experimenting. Most projects will end up implementing some of this functionality themselves over the `starlark` library, incorporating their specific extra types etc. - -In particular the `starlark` binary _can_ be effectively used as a linter. But for the REPL, evaluator and IDE features the `starlark` binary is only aware of standard Starlark. Most Starlark embeddings supply extra functions and data types to work with domain-specific concerns, and the lack of these bindings will cause the REPL/evaluator to fail if they are used, and will give a subpar IDE experience. In most cases you should write your own binary depending on the `starlark` library, integrating your domain-specific pieces, and then using the bundled LSP functions to produce your own IDE/REPL/evaluator on top of those. You should still be able to use the [VS Code extension](vscode/README.md). +There are six components: + +- `starlark_derive`, a proc-macro crate that defines the necessary macros for + Starlark. This library is a dependency of `starlark` the library, which + reexports all the relevant pieces, and should not be used directly. +- `starlark_map`, a library with memory-efficient ordered/unordered maps/sets + and various other data structures useful in Starlark. +- `starlark_syntax`, a library with the AST of Starlark and parsing functions. + Only use if you want to manipulate the AST directly. +- `starlark` the main library, with evaluator, standard library, debugger + support and lots of other pieces. Projects wishing to embed Starlark in their + environment (with additional types, library functions and features) will make + use of this library. This library reexports the relevant pieces of + `starlark_derive`, `starlark_map` and most of `starlark_syntax`. +- `starlark_lsp`, a library providing an + [LSP](https://microsoft.github.io/language-server-protocol/). +- `starlark_bin` the binary, which provides interactive evaluation, IDE features + and linter, exposed through a command line. Useful if you want to use vanilla + Starlark (but if you do, consider Python3 instead) or as a test-bed for + experimenting. Most projects will end up implementing some of this + functionality themselves over the `starlark` and `starlark_lsp` libraries, + incorporating their specific extra types etc. + +In particular the `starlark_bin` binary _can_ be effectively used as a linter. +But for the REPL, evaluator and IDE features the `starlark_bin` binary is only +aware of standard Starlark. Most Starlark embeddings supply extra functions and +data types to work with domain-specific concerns, and the lack of these bindings +will cause the REPL/evaluator to fail if they are used, and will give a subpar +IDE experience. In most cases you should write your own binary depending on the +`starlark` library, integrating your domain-specific pieces, and then using the +bundled LSP functions to produce your own IDE/REPL/evaluator on top of those. +You should still be able to use the [VS Code extension](vscode/README.md). ## Compatibility -In this section we outline where we don't comply with the [Starlark spec](https://github.com/bazelbuild/starlark/blob/master/spec.md). +In this section we outline where we don't comply with the +[Starlark spec](https://github.com/bazelbuild/starlark/blob/master/spec.md). -* We have plenty of extensions, e.g. type annotations, recursion, top-level `for`. -* We don't yet support later additions to Starlark, such as [bytes](https://github.com/facebookexperimental/starlark-rust/issues/4). -* In some cases creating circular data structures may lead to stack overflows. +- We have plenty of extensions, e.g. type annotations, recursion, top-level + `for`. +- We don't yet support later additions to Starlark, such as + [bytes](https://github.com/facebookexperimental/starlark-rust/issues/4). +- In some cases creating circular data structures may lead to stack overflows. ## Making a release -1. Check the [GitHub Actions](https://github.com/facebookexperimental/starlark-rust/actions) are green. -2. Update `CHANGELOG.md` with the changes since the last release. [This link](https://github.com/facebookexperimental/starlark-rust/compare/v0.4.0...main) can help (update to compare against the last release). -3. Update the version numbers of the two `Cargo.toml` files. Bump them by 0.0.1 if there are no incompatible changes, or 0.1.0 if there are. Bump the dependency in `starlark` to point at the latest `starlark_derive` version. -4. Copy the files `CHANGELOG.md`, `LICENSE` and `README.md` into each `starlark` and `starlark_derive` subdirectory. -5. Run `cargo publish --allow-dirty --dry-run`, then without the `--dry-run`, first in `starlark_derive` and then `starlark` directories. -6. Create a [GitHub release](https://github.com/facebookexperimental/starlark-rust/releases/new) with `v0.X.Y`, using the `starlark` version as the name. +1. Check the + [GitHub Actions](https://github.com/facebookexperimental/starlark-rust/actions) + are green. +2. Update `CHANGELOG.md` with the changes since the last release. + [This link](https://github.com/facebookexperimental/starlark-rust/compare/v0.4.0...main) + can help (update to compare against the last release). +3. Update the version numbers of the two `Cargo.toml` files. Bump them by 0.0.1 + if there are no incompatible changes, or 0.1.0 if there are. Bump the + dependency in `starlark` to point at the latest `starlark_derive` version. +4. Copy the files `CHANGELOG.md`, `LICENSE` and `README.md` into each + subdirectory. +5. Run `cargo publish --allow-dirty --dry-run`, then without the `--dry-run`, in + each of the component directories in the [order above](#components). +6. Create a + [GitHub release](https://github.com/facebookexperimental/starlark-rust/releases/new) + with `v0.X.Y`, using the `starlark` version as the name. ## License -Starlark Rust is Apache License, Version 2.0 licensed, as found in the [LICENSE](LICENSE) file. +Starlark Rust is Apache License, Version 2.0 licensed, as found in the +[LICENSE](LICENSE) file. diff --git a/starlark-rust/docs/environment.md b/starlark-rust/docs/environment.md index 9bd80b5d95897..d527d482eda2e 100644 --- a/starlark-rust/docs/environment.md +++ b/starlark-rust/docs/environment.md @@ -1,10 +1,11 @@ # Environments -:::warning -Some of the information within this page is outdated. However, the explanation of the problem, and thought process behind it, remains useful. The storage of values is similar but implemented using different types. -::: +:::warning Some of the information within this page is outdated. However, the +explanation of the problem, and thought process behind it, remains useful. The +storage of values is similar but implemented using different types. ::: -Starlark (with a nested `def`) has a series of environments that may be active during an evaluation, as illustrated in the following example: +Starlark (with a nested `def`) has a series of environments that may be active +during an evaluation, as illustrated in the following example: ```python x = [] @@ -17,21 +18,30 @@ def foo(): The above example features the following environments: -* Global environment - defining things like `list.append` -* Module environment - defining `x` -* Environment of `foo` - defining `y` -* Environment of `bar` - defining `z` +- Global environment - defining things like `list.append` +- Module environment - defining `x` +- Environment of `foo` - defining `y` +- Environment of `bar` - defining `z` -A scope can *access* variables defined above it, and often *mutate* them, but not *assign* them. +A scope can _access_ variables defined above it, and often _mutate_ them, but +not _assign_ them. To unpack that: -* From the statements inside `bar`, you can access `list.append`, `x`, `y`, and `z`. -* From inside `bar`, you can mutate the variables to be accessed with statements like `list.append(x, 1)` (which may also be termed `x.append(1)`). - * However, before this module is imported by another module, all of its exports become *frozen*, which means it isn't possible to mutate a global list, and if `foo` is called from a different module, then `x` can't be modified. -* If `bar` does `x = 1` that defines a local variable `x` in the function `bar`, shadowing the global `x`. As a consequence, you cannot assign to variables defined in an outer scope. - -Note that assignment *after*, or even *in* non-executed conditional branches, introduces a local variable. +- From the statements inside `bar`, you can access `list.append`, `x`, `y`, and + `z`. +- From inside `bar`, you can mutate the variables to be accessed with statements + like `list.append(x, 1)` (which may also be termed `x.append(1)`). + - However, before this module is imported by another module, all of its + exports become _frozen_, which means it isn't possible to mutate a global + list, and if `foo` is called from a different module, then `x` can't be + modified. +- If `bar` does `x = 1` that defines a local variable `x` in the function `bar`, + shadowing the global `x`. As a consequence, you cannot assign to variables + defined in an outer scope. + +Note that assignment _after_, or even _in_ non-executed conditional branches, +introduces a local variable. For example: @@ -43,21 +53,28 @@ def f(): x = 2 ``` -In the above code, on executing `f()`, it would complain that `x` is referenced before assignment, as the assignment `x = 2` makes `x` a local variable. +In the above code, on executing `f()`, it would complain that `x` is referenced +before assignment, as the assignment `x = 2` makes `x` a local variable. -The rest of this document outlines the various types of environments, how they are accessed, and how they are updated. +The rest of this document outlines the various types of environments, how they +are accessed, and how they are updated. ## Global Environment -The global environment is always frozen and consists of *functions* and *type-values*. All things in the global environment are accessed by name. +The global environment is always frozen and consists of _functions_ and +_type-values_. All things in the global environment are accessed by name. -Type-values are things like `list.append`, which is used when you do either `list.append(xs, 1)` or `xs.append(1)`, assuming `xs` is of type `list`. The available methods for a type can be queried (for example, `dir(list)`). +Type-values are things like `list.append`, which is used when you do either +`list.append(xs, 1)` or `xs.append(1)`, assuming `xs` is of type `list`. The +available methods for a type can be queried (for example, `dir(list)`). There are also global functions, such as `len`, `range`, and `str`. ## Slots -To optimise evaluation, all variables are accessed by integers, which are known as 'slots'. Many variables can be converted to slots statically during compilation, and those which can't have their slot looked up by name at runtime. +To optimise evaluation, all variables are accessed by integers, which are known +as 'slots'. Many variables can be converted to slots statically during +compilation, and those which can't have their slot looked up by name at runtime. The `Slots` data type is defined as: @@ -72,25 +89,39 @@ struct FrozenSlots(Arc>>); As featured in the above code: -* A set of slots are either `Frozen`, which came from another module behind `Arc` or just normal `Slots`, which can be manipulated by the current scope (behind a `Rc`/`RefCell` for single-threaded use and mutation). -* `Vec` is accessed by the slot index. -* `Option` refers to whether the slot has been assigned yet (to detect variables referenced before assignment). +- A set of slots are either `Frozen`, which came from another module behind + `Arc` or just normal `Slots`, which can be manipulated by the current scope + (behind a `Rc`/`RefCell` for single-threaded use and mutation). +- `Vec` is accessed by the slot index. +- `Option` refers to whether the slot has been assigned yet (to detect variables + referenced before assignment). ## Module Environment -The module environment is where the module executes, namely where `x` is defined above. The module environment can have values added in the following standards-conforming ways: +The module environment is where the module executes, namely where `x` is defined +above. The module environment can have values added in the following +standards-conforming ways: -* Assignment statements (such as `x = 1` or `x += 1`). -* `For` loops (such as the `x` in `for x in []:`). -* Via the `load("a.bzl", "foo")`, which imports `foo` frozen. -* Via `def foo():`, which defines `foo` in the module environment. Whether a `def` is frozen or not, when it's executed, its local variables are not frozen. +- Assignment statements (such as `x = 1` or `x += 1`). +- `For` loops (such as the `x` in `for x in []:`). +- Via the `load("a.bzl", "foo")`, which imports `foo` frozen. +- Via `def foo():`, which defines `foo` in the module environment. Whether a + `def` is frozen or not, when it's executed, its local variables are not + frozen. -In addition, two non-standards-conforming ways of defining variables are supported: +In addition, two non-standards-conforming ways of defining variables are +supported: -* Some modules can be injected as bindings in advance. Given a module `foo` that is injected, all the bindings of `foo` will be inserted in this module as frozen. -* The function `load_symbols` injects a dictionary of bindings into the module environment. +- Some modules can be injected as bindings in advance. Given a module `foo` that + is injected, all the bindings of `foo` will be inserted in this module as + frozen. +- The function `load_symbols` injects a dictionary of bindings into the module + environment. -Note that a module has a fixed set of variables (from the standards-conforming ways), a pre-execution set (from the injections) and yet more variables at runtime (via `load_symbols`). To support that structure, the mapping from name to slot index is tracked in a struct: +Note that a module has a fixed set of variables (from the standards-conforming +ways), a pre-execution set (from the injections) and yet more variables at +runtime (via `load_symbols`). To support that structure, the mapping from name +to slot index is tracked in a struct: ```rust enum Names { @@ -100,15 +131,23 @@ enum Names { struct FrozenNames(Arc>); ``` -Each name is given an entry in the map with an increasing slot index. A name will only be assigned a slot once, reusing it thereafter. A corresponding `Slots` data type provides the values associated with those names. +Each name is given an entry in the map with an increasing slot index. A name +will only be assigned a slot once, reusing it thereafter. A corresponding +`Slots` data type provides the values associated with those names. -Importantly, the `Slots` can be extended at runtime by the `load_symbols` function. As with `Slots`, you can either share things behind an `Arc` or mutate them behind an `Rc`/`RefCell`. +Importantly, the `Slots` can be extended at runtime by the `load_symbols` +function. As with `Slots`, you can either share things behind an `Arc` or mutate +them behind an `Rc`/`RefCell`. ## Function Environment -A function can have variables introduced via assignments, `for` loops, and parameters. No additional variables can be discovered at runtime, so all names can be erased at compile time. +A function can have variables introduced via assignments, `for` loops, and +parameters. No additional variables can be discovered at runtime, so all names +can be erased at compile time. -A function can also access variables from the functions it is statically nested within, and from the variables at the root of the module. To support this structure, at runtime we pass around the context, defined as: +A function can also access variables from the functions it is statically nested +within, and from the variables at the root of the module. To support this +structure, at runtime we pass around the context, defined as: ```rust struct Context { @@ -117,13 +156,15 @@ struct Context { } ``` -The above code contains the mapping of names for the module and the slots for the module and each function. +The above code contains the mapping of names for the module and the slots for +the module and each function. -When executed, the inner-most `Slots` (at the end of `slots:`) will never be frozen, as that represents the local variables: but any other may be. +When executed, the inner-most `Slots` (at the end of `slots:`) will never be +frozen, as that represents the local variables: but any other may be. When a function value is captured in a frozen module, use `FrozenContext`: -```rust +````rust struct FrozenContext { names: FrozenNames, slots: Vec, @@ -135,21 +176,34 @@ A list comprehension can be defined as: ```python [x for x in [1,2,3]] -``` +```` In the above code: -* The statement defines a variable `x` that is immediately initialised and shadows any other variables `x` in scope. -* The variable `x` cannot be assigned to, other than in the list comprehension, as it only lives inside the comprehension and the comprehension does not permit assignment statements (only expressions). Such names are not available at the top-level, even when defined in the root of a module. +- The statement defines a variable `x` that is immediately initialised and + shadows any other variables `x` in scope. +- The variable `x` cannot be assigned to, other than in the list comprehension, + as it only lives inside the comprehension and the comprehension does not + permit assignment statements (only expressions). Such names are not available + at the top-level, even when defined in the root of a module. -List comprehensions are implemented by adding additional entries into the `Slots` data type. Even when added at the root of a module, such names are not added to `Names`. +List comprehensions are implemented by adding additional entries into the +`Slots` data type. Even when added at the root of a module, such names are not +added to `Names`. ## Optimisations There are a number of optimisations made to the scheme: -* When freezing a `Names` or `Slots` structure, it's important to only freeze a particular mutable variant once, or you duplicate memory unnecessarily. Therefore, the `Slots` to be `Rc)>>` are augmented, and, similarly, the `Names`. - * When `freeze` is called, the original value is consumed, and the `Some` variant is added. - * **Note**: it is unsafe to ever access the slots after the `freeze`. -* Programs can only assign to the inner-most `Slots`, and that slots must always be mutable. Therefore, define a local `Slots` that is always mutable, and a separate AST node for referring to it. - * For modules, it is important that this mutable local `Slots` is *also* in scope since the scope is used to retrieve unknown variables. +- When freezing a `Names` or `Slots` structure, it's important to only freeze a + particular mutable variant once, or you duplicate memory unnecessarily. + Therefore, the `Slots` to be `Rc)>>` are + augmented, and, similarly, the `Names`. + - When `freeze` is called, the original value is consumed, and the `Some` + variant is added. + - **Note**: it is unsafe to ever access the slots after the `freeze`. +- Programs can only assign to the inner-most `Slots`, and that slots must always + be mutable. Therefore, define a local `Slots` that is always mutable, and a + separate AST node for referring to it. + - For modules, it is important that this mutable local `Slots` is _also_ in + scope since the scope is used to retrieve unknown variables. diff --git a/starlark-rust/docs/gc.md b/starlark-rust/docs/gc.md index 324a3a5a82b62..847d4835489ff 100644 --- a/starlark-rust/docs/gc.md +++ b/starlark-rust/docs/gc.md @@ -2,7 +2,14 @@ This page describes a two-space garbage collector that can deal with cycles. -In Starlark, this pattern is used both when doing a real garbage collection, and when freezing. For both cases, it starts out with a memory block, which has pointers referring to things inside it, and ends up with a new memory block with equivalent pointers inside it. However, only pointers reachable from outside the original memory block are available in the new memory block. The garbage collector can deal with cyclic data structures and the time spent is proportional to the amount of live data in the heap (memory that is dropped is not even visited). +In Starlark, this pattern is used both when doing a real garbage collection, and +when freezing. For both cases, it starts out with a memory block, which has +pointers referring to things inside it, and ends up with a new memory block with +equivalent pointers inside it. However, only pointers reachable from outside the +original memory block are available in the new memory block. The garbage +collector can deal with cyclic data structures and the time spent is +proportional to the amount of live data in the heap (memory that is dropped is +not even visited). ## A worked example @@ -14,53 +21,67 @@ Y := Data("hello", X, Y) Z := Data("universe") ``` -All of `X`, `Y` and `Z` are memory locations. The `Y` memory location has both some data of its own (`"hello"`) and two pointers (`X` and `Y` itself). +All of `X`, `Y` and `Z` are memory locations. The `Y` memory location has both +some data of its own (`"hello"`) and two pointers (`X` and `Y` itself). -The pointers from outside the heap into the heap are known as *roots*. +The pointers from outside the heap into the heap are known as _roots_. -Assuming, in the above example, that `Y` is the only root, then, since `Y` is used from outside, `Y` must be moved to the new memory block. Consequently, the data `X` needs to be copied, but `Z` can be dropped. +Assuming, in the above example, that `Y` is the only root, then, since `Y` is +used from outside, `Y` must be moved to the new memory block. Consequently, the +data `X` needs to be copied, but `Z` can be dropped. Following are the required steps for using a garbage collector: -1. To copy `Y`, allocate a value in the new heap `A` with a sentinel value in it (that that sentinel is called a `Blackhole`). Then, turn `Y` into a `Forward(A)` pointer, so that if anyone else in this cycle tries to collect `Y` they immediately "forward" to the new value and the data from `Y` is grabbed so its pointers can be traversed. That results in the following: +1. To copy `Y`, allocate a value in the new heap `A` with a sentinel value in it + (that that sentinel is called a `Blackhole`). Then, turn `Y` into a + `Forward(A)` pointer, so that if anyone else in this cycle tries to collect + `Y` they immediately "forward" to the new value and the data from `Y` is + grabbed so its pointers can be traversed. That results in the following: - ```bash - X := Data("world") - Y := Forward(A) - Z := Data("universe") + ```bash + X := Data("world") + Y := Forward(A) + Z := Data("universe") - A := Blackhole - ``` + A := Blackhole + ``` - With `Data("hello", X, Y)` as the current item being processed. + With `Data("hello", X, Y)` as the current item being processed. -2. Walk the pointers of the current value, performing a garbage collection on each of them. To copy `Y`, it can be seen that `Y` points at a `Forward(A)` node, so there's no need to do anything. To copy `X`, follow the process starting at step 1, but for `X` (which ends up at `B`). Performing that move leads to the following: +2. Walk the pointers of the current value, performing a garbage collection on + each of them. To copy `Y`, it can be seen that `Y` points at a `Forward(A)` + node, so there's no need to do anything. To copy `X`, follow the process + starting at step 1, but for `X` (which ends up at `B`). Performing that move + leads to the following: - ```bash - X := Forward(B) - Y := Forward(A) - Z := Data("universe") + ```bash + X := Forward(B) + Y := Forward(A) + Z := Data("universe") - A := Blackhole - B := Data("world") - ``` + A := Blackhole + B := Data("world") + ``` -3. Replace all the pointers with the forwarded value, and write it back over the `Blackhole` in `A`. This gives the following: +3. Replace all the pointers with the forwarded value, and write it back over the + `Blackhole` in `A`. This gives the following: - ```bash - X := Forward(B) - Y := Forward(A) - Z := Data("universe") + ```bash + X := Forward(B) + Y := Forward(A) + Z := Data("universe") - A := Data("hello", B, A) - B := Data("world") - ``` + A := Data("hello", B, A) + B := Data("world") + ``` -4. Adjust any roots pointing at `Y` to point at `A` and throw away the original heap, which produces the following: +4. Adjust any roots pointing at `Y` to point at `A` and throw away the original + heap, which produces the following: - ```bash - A := Data("hello", B, A) - B := Data("world") - ``` + ```bash + A := Data("hello", B, A) + B := Data("world") + ``` -These above four steps successfully garbage collects a cyclic data structure, while preserving the cycles and getting rid of the unused data. +These above four steps successfully garbage collects a cyclic data structure, +while preserving the cycles and getting rid of the unused data. diff --git a/starlark-rust/docs/heaps.md b/starlark-rust/docs/heaps.md index 22cf8497b1578..0af4f4ebc7b9a 100644 --- a/starlark-rust/docs/heaps.md +++ b/starlark-rust/docs/heaps.md @@ -4,24 +4,36 @@ In Starlark, there are three interesting heap-related points of interest: -* A `Heap` has `Value`'s allocated on it and cannot be cloned or shared. -* A `FrozenHeap` has `FrozenValue`'s allocated on it and cannot be cloned or shared. -* A `FrozenHeapRef` is a `FrozenHeap` that is now read-only and can now be cloned and shared. +- A `Heap` has `Value`'s allocated on it and cannot be cloned or shared. +- A `FrozenHeap` has `FrozenValue`'s allocated on it and cannot be cloned or + shared. +- A `FrozenHeapRef` is a `FrozenHeap` that is now read-only and can now be + cloned and shared. -A `FrozenHeapRef` keeps a heap alive. While you have a `FrozenValue`, it is important that you have either the `FrozenHeap` itself, or more usually, a `FrozenHeapRef` to it. A `FrozenHeap` may contains a set of `FrozenHeapRef`'s to keep the `FrozenHeap`s it references alive. +A `FrozenHeapRef` keeps a heap alive. While you have a `FrozenValue`, it is +important that you have either the `FrozenHeap` itself, or more usually, a +`FrozenHeapRef` to it. A `FrozenHeap` may contains a set of `FrozenHeapRef`'s to +keep the `FrozenHeap`s it references alive. ## Heap Containers Heaps are included in other data types: -* A `Module` contains a `Heap` (where normal values are allocated) and a `FrozenHeap` (stores references to other frozen heaps and has compilation constants allocated on it). The `Heap` portion is garbage collected. At the end, when you call `freeze`, `Value`'s referenced by name in the `Module` are moved to the `FrozenHeap` and then then `FrozenHeap` is sealed to produce a `FrozenHeapRef`. -* A `FrozenModule` contains a `FrozenHeapRef`. -* A `GlobalsBuilder` contains a `FrozenHeap` onto which values are allocated. -* A `Globals` contains a `FrozenHeapRef`. +- A `Module` contains a `Heap` (where normal values are allocated) and a + `FrozenHeap` (stores references to other frozen heaps and has compilation + constants allocated on it). The `Heap` portion is garbage collected. At the + end, when you call `freeze`, `Value`'s referenced by name in the `Module` are + moved to the `FrozenHeap` and then then `FrozenHeap` is sealed to produce a + `FrozenHeapRef`. +- A `FrozenModule` contains a `FrozenHeapRef`. +- A `GlobalsBuilder` contains a `FrozenHeap` onto which values are allocated. +- A `Globals` contains a `FrozenHeapRef`. ## Heap References -It is important that when a `FrozenValue` X is referenced by a `Value` or `FrozenValue` (for example, included in a list), the heap where X originates is added as a reference to the heap where the new value is being created. +It is important that when a `FrozenValue` X is referenced by a `Value` or +`FrozenValue` (for example, included in a list), the heap where X originates is +added as a reference to the heap where the new value is being created. As a concrete example in pseudo-code: @@ -40,19 +52,29 @@ In the above code, the following steps are taken: 1. Create a `FrozenHeap` then allocate something in it. 1. Turn the heap into a reference. 1. Use the allocated value `s` from `h1` when constructing a value in `h2`. -1. For that to be legal, and for the heap `h1` to not disappear while it is being allocated, it is important to call `add_reference`. +1. For that to be legal, and for the heap `h1` to not disappear while it is + being allocated, it is important to call `add_reference`. -Note that this API can only point at a `FrozenValue` from another heap, and only after that heap has been turned into a reference, so it will not be allocated in anymore. These restrictions are deliberate and mean that most programs only have one 'active heap' at a time. +Note that this API can only point at a `FrozenValue` from another heap, and only +after that heap has been turned into a reference, so it will not be allocated in +anymore. These restrictions are deliberate and mean that most programs only have +one 'active heap' at a time. Following are some places where heap references are added by Starlark: -* Before evaluation is started, a reference is added to the `Globals` from the `Module`, so it can access the global functions. -* When evaluating a `load` statement, a reference is added to the `FrozenModule` that is being loaded. -* When freezing a module, the `FrozenHeap`, in the `Module`, is moved to the `FrozenModule`, preserving the references that were added. +- Before evaluation is started, a reference is added to the `Globals` from the + `Module`, so it can access the global functions. +- When evaluating a `load` statement, a reference is added to the `FrozenModule` + that is being loaded. +- When freezing a module, the `FrozenHeap`, in the `Module`, is moved to the + `FrozenModule`, preserving the references that were added. ## `OwnedFrozenValue` -When you get a value from a `FrozenModule`, it will be a `OwnedFrozenValue`. This structure is a pair of a `FrozenHeapRef` and a `FrozenValue`, where the ref keeps the value alive. You can move that `OwnedFrozenValue` into the value of a module with code such as: +When you get a value from a `FrozenModule`, it will be a `OwnedFrozenValue`. +This structure is a pair of a `FrozenHeapRef` and a `FrozenValue`, where the ref +keeps the value alive. You can move that `OwnedFrozenValue` into the value of a +module with code such as: ```rust fn move<'v>(from: &FrozenModule, to: &'v Module) { @@ -64,9 +86,16 @@ fn move<'v>(from: &FrozenModule, to: &'v Module) { In general, you can use the `OwnedFrozenValue` in one of three ways: -* **Operate on it directly** - with methods like `unpack_i32` or `to_str`. -* **Extract it safely** - using methods like `owned_frozen_value`, which takes a `FrozenHeap` to which the heap reference is added and returns a naked `FrozenValue`. After that, it is then safe for the `FrozenHeap` you passed in to use the `FrozenValue`. - * With `owned_value`, there is lifetime checking that the right heap is passed, but with `FrozenValue`, there isn't. - * Be careful to pass the right heap, although given most programs only have one active heap at a time, it should mostly work out. -* **Extract it unsafely** - using methods `unchecked_frozen_value`, which gives you the underlying `FrozenValue` without adding any references. - * Be careful to make sure there is a good reason the `FrozenValue` remains valid. +- **Operate on it directly** - with methods like `unpack_i32` or `to_str`. +- **Extract it safely** - using methods like `owned_frozen_value`, which takes a + `FrozenHeap` to which the heap reference is added and returns a naked + `FrozenValue`. After that, it is then safe for the `FrozenHeap` you passed in + to use the `FrozenValue`. + - With `owned_value`, there is lifetime checking that the right heap is + passed, but with `FrozenValue`, there isn't. + - Be careful to pass the right heap, although given most programs only have + one active heap at a time, it should mostly work out. +- **Extract it unsafely** - using methods `unchecked_frozen_value`, which gives + you the underlying `FrozenValue` without adding any references. + - Be careful to make sure there is a good reason the `FrozenValue` remains + valid. diff --git a/starlark-rust/docs/types.md b/starlark-rust/docs/types.md index 009c862858fa5..c50b20efabc88 100644 --- a/starlark-rust/docs/types.md +++ b/starlark-rust/docs/types.md @@ -1,6 +1,7 @@ # Starlark Types -The Starlark 'types' extension is highly experimental and likely to be modified in the future. +The Starlark 'types' extension is highly experimental and likely to be modified +in the future. Types can be added to function arguments, or function return types. @@ -13,46 +14,68 @@ def fib(i: int) -> int: There are moments where types can be checked: -1. At runtime, as a function is executed, when a value of the appropriate type is available. +1. At runtime, as a function is executed, when a value of the appropriate type + is available. 2. Statically, without executing anything. -3. At compile time, when the definitions of all symbols imported using `load` are available. +3. At compile time, when the definitions of all symbols imported using `load` + are available. -Currently runtime is the normal way of checking, but other systems built on Starlark (e.g. Buck2) may also perform additional types of checking. In all cases the meaning of the types is the same. +Currently runtime is the normal way of checking, but other systems built on +Starlark (e.g. Buck2) may also perform additional types of checking. In all +cases the meaning of the types is the same. -The rest of this document lays out what types mean and what type-supporting values are available (records and enums). +The rest of this document lays out what types mean and what type-supporting +values are available (records and enums). ## What does a type mean? A type is a Starlark expression that has a meaning as a type: -* When `fib(3)` is called, the *value* `3` is passed to `fib` as parameter `i`. -* When the execution of `fib` is started, the *expression* `int` is evaluated to the value of the `int` function. -* A check is then made that the value `3` matches the type represented by `int`. +- When `fib(3)` is called, the _value_ `3` is passed to `fib` as parameter `i`. +- When the execution of `fib` is started, the _expression_ `int` is evaluated to + the value of the `int` function. +- A check is then made that the value `3` matches the type represented by `int`. -If the value doesn't match, it is a runtime error. Similarly, on `return` statements, or the end of the function, a check is made that result type matches `int`. +If the value doesn't match, it is a runtime error. Similarly, on `return` +statements, or the end of the function, a check is made that result type matches +`int`. As some examples of types: -* The type `typing.Any` matches any value, with no restrictions. -* The types `int`, `bool`, `str` all represent the values produced by the respective functions. -* The type `None` represents the value `None`. -* The type `list[int]` represents a list of `int` types, e.g. `list[typing.Any]` represents a list containing any types. -* The type `dict[int, bool]` represents a dictionary with `int` keys and `bool` values. -* The type `tuple[int, bool, str]` represents a tuple of arity 3 with components being `int`, `bool` and `str`. -* The type `tuple[int, ...]` represents a tuple of unknown arity where all the components are of type `int`. -* The type `int | bool` represents a value that is either an `int` or a `bool`. -* The type `typing.Callable` represents something that can be called as a function. -* The type `typing.Iterable` represents something that can be iterated on. -* The type `typing.Never` represents a type with no valid values - e.g. the result of `fail` is `typing.Never` as the return value of `fail` can never be observed, given the program terminates. +- The type `typing.Any` matches any value, with no restrictions. +- The types `int`, `bool`, `str` all represent the values produced by the + respective functions. +- The type `None` represents the value `None`. +- The type `list[int]` represents a list of `int` types, e.g. `list[typing.Any]` + represents a list containing any types. +- The type `dict[int, bool]` represents a dictionary with `int` keys and `bool` + values. +- The type `tuple[int, bool, str]` represents a tuple of arity 3 with components + being `int`, `bool` and `str`. +- The type `tuple[int, ...]` represents a tuple of unknown arity where all the + components are of type `int`. +- The type `int | bool` represents a value that is either an `int` or a `bool`. +- The type `typing.Callable` represents something that can be called as a + function. +- The type `typing.Iterable` represents something that can be iterated on. +- The type `typing.Never` represents a type with no valid values - e.g. the + result of `fail` is `typing.Never` as the return value of `fail` can never be + observed, given the program terminates. The goals of this type system are: -* Reuse the existing machinery of Starlark as much as possible, avoiding inventing a special class of type values. As a consequence, any optimisations for values like string/list are reused. -* Provide a pleasing syntax. -* Some degree of compatibility with Python, which allows types as expressions in the same places Buck2 allows them (but with different meaning and different checking). -* And finally, a non-goal is to provide a complete type system capable of representing every type invariant: it's intended to be a lossy approximation. +- Reuse the existing machinery of Starlark as much as possible, avoiding + inventing a special class of type values. As a consequence, any optimisations + for values like string/list are reused. +- Provide a pleasing syntax. +- Some degree of compatibility with Python, which allows types as expressions in + the same places Buck2 allows them (but with different meaning and different + checking). +- And finally, a non-goal is to provide a complete type system capable of + representing every type invariant: it's intended to be a lossy approximation. -In addition to these built-in types, records and enumerations are provided as special concepts. +In addition to these built-in types, records and enumerations are provided as +special concepts. ## Record types @@ -64,15 +87,22 @@ For example: MyRecord = record(host=str, port=int) ``` -This above statement defines a record `MyRecord` with 2 fields, the first named `host` that must be of type `str`, and the second named `port` that must be of type `int`. +This above statement defines a record `MyRecord` with 2 fields, the first named +`host` that must be of type `str`, and the second named `port` that must be of +type `int`. Now `MyRecord` is defined, it's possible to do the following: -* Create values of this type with `MyRecord(host="localhost", port=80)`. It is a runtime error if any arguments are missed, of the wrong type, or if any unexpected arguments are given. -* Get the type of the record suitable for a type annotation with `MyRecord`. -* Get the fields of the record. For example, `v = MyRecord(host="localhost", port=80)` will provide `v.host == "localhost"` and `v.port == 80`. Similarly, `dir(v) == ["host", "port"]`. +- Create values of this type with `MyRecord(host="localhost", port=80)`. It is a + runtime error if any arguments are missed, of the wrong type, or if any + unexpected arguments are given. +- Get the type of the record suitable for a type annotation with `MyRecord`. +- Get the fields of the record. For example, + `v = MyRecord(host="localhost", port=80)` will provide `v.host == "localhost"` + and `v.port == 80`. Similarly, `dir(v) == ["host", "port"]`. -It is also possible to specify default values for parameters using the `field` function. +It is also possible to specify default values for parameters using the `field` +function. For example: @@ -80,9 +110,11 @@ For example: MyRecord = record(host=str, port=field(int, 80)) ``` -Now the `port` field can be omitted, defaulting to `80` is not present (for example, `MyRecord(host="localhost").port == 80`). +Now the `port` field can be omitted, defaulting to `80` is not present (for +example, `MyRecord(host="localhost").port == 80`). -Records are stored deduplicating their field names, making them more memory efficient than dictionaries. +Records are stored deduplicating their field names, making them more memory +efficient than dictionaries. ## Enum types @@ -94,14 +126,22 @@ For example: MyEnum = enum("option1", "option2", "option3") ``` -This statement defines an enumeration `MyEnum` that consists of the three values `"option1"`, `"option2"` and `"option3"`. +This statement defines an enumeration `MyEnum` that consists of the three values +`"option1"`, `"option2"` and `"option3"`. Now `MyEnum` is defined, it's possible to do the following: -* Create values of this type with `MyEnum("option2")`. It is a runtime error if the argument is not one of the predeclared values of the enumeration. -* Get the type of the enum suitable for a type annotation with `MyEnum`. -* Given a value of the enum (for example, `v = MyEnum("option2")`), get the underlying value `v.value == "option2"` or the index in the enumeration `v.index == 1`. -* Get a list of the values that make up the array with `MyEnum.values() == ["option1", "option2", "option3"]`. -* Treat `MyEnum` a bit like an array, with `len(MyEnum) == 3`, `MyEnum[1] == MyEnum("option2")` and iteration over enums `[x.value for x in MyEnum] == ["option1", "option2", "option3"]`. - -Enumeration types store each value once, which are then efficiently referenced by enumeration values. +- Create values of this type with `MyEnum("option2")`. It is a runtime error if + the argument is not one of the predeclared values of the enumeration. +- Get the type of the enum suitable for a type annotation with `MyEnum`. +- Given a value of the enum (for example, `v = MyEnum("option2")`), get the + underlying value `v.value == "option2"` or the index in the enumeration + `v.index == 1`. +- Get a list of the values that make up the array with + `MyEnum.values() == ["option1", "option2", "option3"]`. +- Treat `MyEnum` a bit like an array, with `len(MyEnum) == 3`, + `MyEnum[1] == MyEnum("option2")` and iteration over enums + `[x.value for x in MyEnum] == ["option1", "option2", "option3"]`. + +Enumeration types store each value once, which are then efficiently referenced +by enumeration values. diff --git a/starlark-rust/docs/values.md b/starlark-rust/docs/values.md index 7938faa1e09e1..6bc7ae0a54e03 100644 --- a/starlark-rust/docs/values.md +++ b/starlark-rust/docs/values.md @@ -1,21 +1,29 @@ # Value Representation -:::warning -Some of the information in this page is outdated. However, the explanation of the problem, and thought process behind it, remains useful. Of particular note is that a garbage collected heap is now used for `Value`. -::: +:::warning Some of the information in this page is outdated. However, the +explanation of the problem, and thought process behind it, remains useful. Of +particular note is that a garbage collected heap is now used for `Value`. ::: -This page explains how values are represented in the Starlark interpreter, ignoring some incidental details. +This page explains how values are represented in the Starlark interpreter, +ignoring some incidental details. -Importantly, in Starlark, any identifiers from modules that you import are 'frozen', which means that, if you have a module that defines a list, then once you have imported the module, the list is now immutable. This design means that you can safely share imports with multiple users, without any expensive copying, and use the imports in parallel. +Importantly, in Starlark, any identifiers from modules that you import are +'frozen', which means that, if you have a module that defines a list, then once +you have imported the module, the list is now immutable. This design means that +you can safely share imports with multiple users, without any expensive copying, +and use the imports in parallel. ## Frozen vs unfrozen values Values that are frozen are segregated from those that are not: -* Frozen values are those you import, and (assuming no GC) are to be ref-counted atomically (so they can be shared by multiple threads) and never changed. -* Unfrozen values are those which are local to the module, and, since modules execute single threaded, can be non-atomically ref-counted and mutated. +- Frozen values are those you import, and (assuming no GC) are to be ref-counted + atomically (so they can be shared by multiple threads) and never changed. +- Unfrozen values are those which are local to the module, and, since modules + execute single threaded, can be non-atomically ref-counted and mutated. -Once a module has finished executing, it's values are frozen and can be reused freely. +Once a module has finished executing, it's values are frozen and can be reused +freely. ## Thaw-on-write @@ -28,11 +36,19 @@ def my_list(x): return ([1,2,3], x) ``` -This above code returns the unfrozen list `[1,2,3]`. But while the list is unfrozen, and could be mutated by the caller, it probably won't be. To optimise this pattern, construct a frozen list when compiling `my_list` and insert a shared reference to it in the result. If anyone tries to mutate the list, it's explicitly unfrozen by copying it into a mutable variant (known as thawing the value). +This above code returns the unfrozen list `[1,2,3]`. But while the list is +unfrozen, and could be mutated by the caller, it probably won't be. To optimise +this pattern, construct a frozen list when compiling `my_list` and insert a +shared reference to it in the result. If anyone tries to mutate the list, it's +explicitly unfrozen by copying it into a mutable variant (known as thawing the +value). ## Immutable containers of mutable data -There are some data types (such as functions and tuples) that are themselves immutable but contain mutable data. Importantly, all types that can be invoked as functions (for example, `lambda`, `def`, and `a.b()`) fall into this category. These types can be non-atomically ref-counted but can't be mutated. +There are some data types (such as functions and tuples) that are themselves +immutable but contain mutable data. Importantly, all types that can be invoked +as functions (for example, `lambda`, `def`, and `a.b()`) fall into this +category. These types can be non-atomically ref-counted but can't be mutated. ## Implementation in Rust @@ -58,16 +74,18 @@ enum Mutable { } ``` -In the above code, both of the traits `dyn SimpleValue` `and dyn ComplexValue` enable you to convert to the other and have shared general value-like methods. +In the above code, both of the traits `dyn SimpleValue` `and dyn ComplexValue` +enable you to convert to the other and have shared general value-like methods. There are four types of value: -* `Immutable` -* `Pseudo` - immutable containers of mutable values. -* `Mutable`/`Mutable` -* `Mutable`/`ThawOnWrite` - immutable now but can be replaced with `Mutable`/`Mutable` if needed. +- `Immutable` +- `Pseudo` - immutable containers of mutable values. +- `Mutable`/`Mutable` +- `Mutable`/`ThawOnWrite` - immutable now but can be replaced with + `Mutable`/`Mutable` if needed. There are two root types: -* `FrozenValue` - imported. -* `Value` - defined locally. +- `FrozenValue` - imported. +- `Value` - defined locally. diff --git a/starlark-rust/starlark/Cargo.toml b/starlark-rust/starlark/Cargo.toml index 15cb312c444fc..5abe23942247c 100644 --- a/starlark-rust/starlark/Cargo.toml +++ b/starlark-rust/starlark/Cargo.toml @@ -13,7 +13,7 @@ keywords = ["starlark", "skylark", "bazel", "language", "interpreter"] license = "Apache-2.0" name = "starlark" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [dependencies] anyhow = "1.0.65" @@ -25,7 +25,7 @@ display_container = { workspace = true } dupe = { workspace = true } either = "1.8" erased-serde = "0.3.12" -hashbrown = { version = "0.12.3", features = ["raw"] } +hashbrown = { version = "0.14.3", features = ["raw"] } inventory = "0.3.8" itertools = "0.10" maplit = "1.0.2" @@ -37,9 +37,9 @@ paste = "1.0" regex = "1.5.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -starlark_derive = { version = "0.10.0", path = "../starlark_derive" } -starlark_map = { version = "0.10.0", path = "../starlark_map" } -starlark_syntax = { version = "0.10.0", path = "../starlark_syntax" } +starlark_derive = { version = "0.12.0", path = "../starlark_derive" } +starlark_map = { version = "0.12.0", path = "../starlark_map" } +starlark_syntax = { version = "0.12.0", path = "../starlark_syntax" } static_assertions = "1.1.0" strsim = "0.10.0" textwrap = "0.11" diff --git a/starlark-rust/starlark/src/analysis/find_call_name.rs b/starlark-rust/starlark/src/analysis/find_call_name.rs index fa3c34933f55e..ea09695741a19 100644 --- a/starlark-rust/starlark/src/analysis/find_call_name.rs +++ b/starlark-rust/starlark/src/analysis/find_call_name.rs @@ -79,7 +79,7 @@ impl AstModuleFindCallName for AstModule { } #[cfg(test)] -mod test { +mod tests { use starlark_syntax::syntax::module::AstModuleFields; use crate::analysis::find_call_name::AstModuleFindCallName; diff --git a/starlark-rust/starlark/src/any.rs b/starlark-rust/starlark/src/any.rs index 42a0bb45c6980..90fa9b382b6af 100644 --- a/starlark-rust/starlark/src/any.rs +++ b/starlark-rust/starlark/src/any.rs @@ -86,9 +86,9 @@ unsafe impl<'a, T: ProvidesStaticType<'a> + 'a + ?Sized> AnyLifetime<'a> for T { /// struct Baz(T); /// # // TODO: `#[derive(ProvidesStaticType)]` should learn to handle this case too. /// unsafe impl<'a, T> ProvidesStaticType<'a> for Baz -/// where -/// T: ProvidesStaticType<'a> + Display, -/// T::StaticType: Display + Sized, +/// where +/// T: ProvidesStaticType<'a> + Display, +/// T::StaticType: Display + Sized, /// { /// type StaticType = Baz; /// } diff --git a/starlark-rust/starlark/src/assert/assert.rs b/starlark-rust/starlark/src/assert/assert.rs index 0cbcb127ac244..3084e5e0e2f50 100644 --- a/starlark-rust/starlark/src/assert/assert.rs +++ b/starlark-rust/starlark/src/assert/assert.rs @@ -447,7 +447,7 @@ impl<'a> Assert<'a> { if !err_msg.contains(msg) { original.eprint(); panic!( - "starlark::assert::{}, failed with the wrong message!\nCode:\n{}\nError:\n{}\nMissing:\n{}\nExpected:\n{:?}", + "starlark::assert::{}, failed with the wrong message!\nCode:\n{}\nError:\n{:#}\nMissing:\n{}\nExpected:\n{:?}", func, program, inner, msg, msgs ) } @@ -520,10 +520,12 @@ impl<'a> Assert<'a> { /// /// ``` /// # use starlark::assert::Assert; - /// Assert::new().is_true(r#" + /// Assert::new().is_true( + /// r#" /// x = 1 + 1 /// x == 2 - /// "#); + /// "#, + /// ); /// ``` pub fn is_true(&self, program: &str) { self.with_gc(|gc| { @@ -544,11 +546,13 @@ impl<'a> Assert<'a> { /// /// ``` /// # use starlark::assert::Assert; - /// Assert::new().all_true(r#" + /// Assert::new().all_true( + /// r#" /// 1 == 1 /// /// 2 == 1 + 1 - /// "#); + /// "#, + /// ); /// ``` pub fn all_true(&self, program: &str) { self.with_gc(|gc| { diff --git a/starlark-rust/starlark/src/assert/mod.rs b/starlark-rust/starlark/src/assert/mod.rs index e3da74c30c077..72d2ca3599898 100644 --- a/starlark-rust/starlark/src/assert/mod.rs +++ b/starlark-rust/starlark/src/assert/mod.rs @@ -45,4 +45,3 @@ mod assert; mod conformance; pub use assert::*; -pub use conformance::*; diff --git a/starlark-rust/starlark/src/coerce.rs b/starlark-rust/starlark/src/coerce.rs index a146641c8edc6..8f8fe22aef5cb 100644 --- a/starlark-rust/starlark/src/coerce.rs +++ b/starlark-rust/starlark/src/coerce.rs @@ -42,16 +42,14 @@ use starlark_map::small_map::SmallMap; /// One use of `Coerce` is around newtype wrappers: /// /// ``` -/// use starlark::coerce::{Coerce, coerce}; +/// use starlark::coerce::coerce; +/// use starlark::coerce::Coerce; /// #[repr(transparent)] /// #[derive(Debug, Coerce)] /// struct Wrapper(String); /// /// let value = vec![Wrapper("hello".to_owned()), Wrapper("world".to_owned())]; -/// assert_eq!( -/// coerce::<_, &Vec>(&value).join(" "), -/// "hello world" -/// ); +/// assert_eq!(coerce::<_, &Vec>(&value).join(" "), "hello world"); /// let mut value = coerce::<_, Vec>(value); /// assert_eq!(value.pop(), Some("world".to_owned())); /// ``` @@ -59,7 +57,8 @@ use starlark_map::small_map::SmallMap; /// Another involves containers: /// /// ``` -/// use starlark::coerce::{Coerce, coerce}; +/// use starlark::coerce::coerce; +/// use starlark::coerce::Coerce; /// # #[derive(Coerce)] /// # #[repr(transparent)] /// # struct Wrapper(String); @@ -68,10 +67,7 @@ use starlark_map::small_map::SmallMap; /// struct Container(i32, T); /// /// let value = Container(20, Wrapper("twenty".to_owned())); -/// assert_eq!( -/// coerce::<_, &Container>(&value).1, -/// "twenty" -/// ); +/// assert_eq!(coerce::<_, &Container>(&value).1, "twenty"); /// ``` /// /// If you only need [`coerce`] on newtype references, diff --git a/starlark-rust/starlark/src/docs/markdown.rs b/starlark-rust/starlark/src/docs/markdown.rs index 31f60b150ce4e..b9d2d4bd8b70f 100644 --- a/starlark-rust/starlark/src/docs/markdown.rs +++ b/starlark-rust/starlark/src/docs/markdown.rs @@ -15,6 +15,7 @@ * limitations under the License. */ +use std::fmt::Write; use std::slice; use dupe::Dupe; @@ -143,11 +144,19 @@ fn render_function_parameters(params: &[DocParam]) -> Option { DocParam::Args { name, docs, .. } => Some((name, docs)), DocParam::Kwargs { name, docs, .. } => Some((name, docs)), }) - .map(|(name, docs)| { + .fold(String::new(), |mut output, (name, docs)| { let docs = render_doc_string(DSOpts::Combined, docs).unwrap_or_default(); - format!("* `{name}`: {docs}\n") - }) - .collect(); + + let mut lines_iter = docs.lines(); + let first_line = lines_iter.next().unwrap(); + let rest_of_lines: Vec<&str> = lines_iter.collect(); + + let _ = writeln!(output, "* `{name}`: {first_line}"); + for line in &rest_of_lines { + let _ = writeln!(output, " {line}"); + } + output + }); Some(param_list) } diff --git a/starlark-rust/starlark/src/eval/bc/stack_ptr.rs b/starlark-rust/starlark/src/eval/bc/stack_ptr.rs index 1efa546f709af..4ab27efb44d2e 100644 --- a/starlark-rust/starlark/src/eval/bc/stack_ptr.rs +++ b/starlark-rust/starlark/src/eval/bc/stack_ptr.rs @@ -103,9 +103,6 @@ impl BcSlotRange { } } -#[derive(Copy, Clone, Dupe, Debug)] -pub(crate) struct BcSlotRangeFrom(pub(crate) BcSlot); - /// Slot containing a value. /// /// The slot may be a local variable, so this slot cannot be used to store a temporary value. diff --git a/starlark-rust/starlark/src/eval/compiler/def.rs b/starlark-rust/starlark/src/eval/compiler/def.rs index d61f6bdade42a..a1da93a27f33e 100644 --- a/starlark-rust/starlark/src/eval/compiler/def.rs +++ b/starlark-rust/starlark/src/eval/compiler/def.rs @@ -33,6 +33,7 @@ use once_cell::sync::Lazy; use starlark_derive::starlark_value; use starlark_derive::NoSerialize; use starlark_derive::VisitSpanMut; +use starlark_map::StarlarkHasher; use starlark_syntax::eval_exception::EvalException; use starlark_syntax::slice_vec_ext::SliceExt; use starlark_syntax::syntax::def::DefParam; @@ -659,6 +660,11 @@ where fn typechecker_ty(&self) -> Option { Some(self.def_info.ty.clone()) } + + fn write_hash(&self, hasher: &mut StarlarkHasher) -> crate::Result<()> { + // It's hard to come up with a good hash here, but let's at least make an effort. + self.def_info.name.write_hash(hasher) + } } impl<'v, V: ValueLike<'v>> DefGen diff --git a/starlark-rust/starlark/src/eval/compiler/scope/mod.rs b/starlark-rust/starlark/src/eval/compiler/scope/mod.rs index 6d55a3eaf3482..3093c7d4aa306 100644 --- a/starlark-rust/starlark/src/eval/compiler/scope/mod.rs +++ b/starlark-rust/starlark/src/eval/compiler/scope/mod.rs @@ -658,7 +658,7 @@ impl<'f> ModuleScopeBuilder<'f> { fn variable_not_found_err(&self, ident: &CstIdent) -> EvalException { let variants = self .current_scope_all_visible_names_for_did_you_mean() - .unwrap_or(Vec::new()); + .unwrap_or_default(); let better = did_you_mean( ident.node.ident.as_str(), variants.iter().map(|s| s.as_str()), diff --git a/starlark-rust/starlark/src/eval/mod.rs b/starlark-rust/starlark/src/eval/mod.rs index 0dd7374d03f0b..1c50441fbbd12 100644 --- a/starlark-rust/starlark/src/eval/mod.rs +++ b/starlark-rust/starlark/src/eval/mod.rs @@ -52,6 +52,7 @@ use crate::eval::compiler::scope::ScopeId; use crate::eval::compiler::Compiler; use crate::eval::runtime::arguments::ArgNames; use crate::eval::runtime::arguments::ArgumentsFull; +use crate::eval::runtime::evaluator; use crate::syntax::DialectTypes; use crate::values::Value; @@ -112,6 +113,11 @@ impl<'v, 'a> Evaluator<'v, 'a> { )), ); + self.call_stack.alloc_if_needed( + self.max_callstack_size + .unwrap_or(evaluator::DEFAULT_STACK_SIZE), + )?; + // Set up the world to allow evaluation (do NOT use ? from now on) self.call_stack.push(Value::new_none(), None).unwrap(); @@ -157,6 +163,10 @@ impl<'v, 'a> Evaluator<'v, 'a> { args: None, kwargs: None, }); + self.call_stack.alloc_if_needed( + self.max_callstack_size + .unwrap_or(evaluator::DEFAULT_STACK_SIZE), + )?; // eval_module pushes an "empty" call stack frame. other places expect that first frame to be ignorable, and // so we push an empty frame too (otherwise things would ignore this function's own frame). self.with_call_stack(Value::new_none(), None, |this| { diff --git a/starlark-rust/starlark/src/eval/runtime/cheap_call_stack.rs b/starlark-rust/starlark/src/eval/runtime/cheap_call_stack.rs index 292edf0c00569..7e9d4fed51a8c 100644 --- a/starlark-rust/starlark/src/eval/runtime/cheap_call_stack.rs +++ b/starlark-rust/starlark/src/eval/runtime/cheap_call_stack.rs @@ -17,10 +17,12 @@ use std::fmt; use std::fmt::Debug; +use std::vec; use dupe::Dupe; use starlark_syntax::codemap::FileSpan; use starlark_syntax::slice_vec_ext::SliceExt; +use starlark_syntax::ErrorKind; use crate::errors::Frame; use crate::eval::runtime::frame_span::FrameSpan; @@ -76,13 +78,15 @@ enum CallStackError { StackIsTooShallowForNthTopFrame(usize, usize), #[error("Starlark call stack overflow")] Overflow, + #[error("Starlark call stack is already allocated")] + AlreadyAllocated, } /// Starlark call stack. #[derive(Debug)] pub(crate) struct CheapCallStack<'v> { count: usize, - stack: Box<[CheapFrame<'v>; MAX_CALLSTACK_RECURSION]>, + stack: Box<[CheapFrame<'v>]>, } impl<'v> Default for CheapCallStack<'v> { @@ -93,25 +97,12 @@ impl<'v> Default for CheapCallStack<'v> { [CheapFrame { function: Value::new_none(), span: None, - }; MAX_CALLSTACK_RECURSION], + }; 0], ), } } } -// Currently, each frame typically allocates about 1K of native stack size (see `test_frame_size`), -// but it is a bit more complicated: -// * each for loop in a frame allocates more native stack -// * inlined functions do not allocate native stack -// Practically max call stack depends on native stack size, -// and depending on environment, it may be configured differently, for example: -// * macOS default stack size is 512KB -// * Linux default stack size is 8MB -// * [tokio default stack size is 2MB][1] -// [1] https://docs.rs/tokio/0.2.1/tokio/runtime/struct.Builder.html#method.thread_stack_size -// TODO(nga): make it configurable. -const MAX_CALLSTACK_RECURSION: usize = 50; - unsafe impl<'v> Trace<'v> for CheapCallStack<'v> { fn trace(&mut self, tracer: &Tracer<'v>) { let (used, unused) = self.stack.split_at_mut(self.count); @@ -128,15 +119,47 @@ unsafe impl<'v> Trace<'v> for CheapCallStack<'v> { } impl<'v> CheapCallStack<'v> { + // Currently, each frame typically allocates about 1K of native stack size (see `test_frame_size`), + // but it is a bit more complicated: + // * each for loop in a frame allocates more native stack + // * inlined functions do not allocate native stack + // Practically max call stack depends on native stack size, + // and depending on environment, it may be configured differently, for example: + // * macOS default stack size is 512KB + // * Linux default stack size is 8MB + // * [tokio default stack size is 2MB][1] + // [1] https://docs.rs/tokio/0.2.1/tokio/runtime/struct.Builder.html#method.thread_stack_size + pub(crate) fn alloc_if_needed(&mut self, max_size: usize) -> anyhow::Result<()> { + if self.stack.len() != 0 { + return if self.stack.len() == max_size { + Ok(()) + } else { + Err(CallStackError::AlreadyAllocated.into()) + }; + } + + self.stack = vec![ + CheapFrame { + function: Value::new_none(), + span: None, + }; + max_size + ] + .into_boxed_slice(); + Ok(()) + } + /// Push an element to the stack. It is important the each `push` is paired /// with a `pop`. pub(crate) fn push( &mut self, function: Value<'v>, span: Option>, - ) -> anyhow::Result<()> { - if unlikely(self.count >= MAX_CALLSTACK_RECURSION) { - return Err(CallStackError::Overflow.into()); + ) -> crate::Result<()> { + if unlikely(self.count >= self.stack.len()) { + return Err(crate::Error::new(ErrorKind::StackOverflow( + CallStackError::Overflow.into(), + ))); } self.stack[self.count] = CheapFrame { function, span }; self.count += 1; diff --git a/starlark-rust/starlark/src/eval/runtime/evaluator.rs b/starlark-rust/starlark/src/eval/runtime/evaluator.rs index ddba3267ee02d..70a91f2e96862 100644 --- a/starlark-rust/starlark/src/eval/runtime/evaluator.rs +++ b/starlark-rust/starlark/src/eval/runtime/evaluator.rs @@ -101,11 +101,18 @@ enum EvaluatorError { CoverageNotEnabled, #[error("Local variable `{0}` referenced before assignment")] LocalVariableReferencedBeforeAssignment(String), + #[error("Max callstack size is already set")] + CallstackSizeAlreadySet, + #[error("Max callstack size cannot be zero")] + ZeroCallstackSize, } /// Number of bytes to allocate between GC's. pub(crate) const GC_THRESHOLD: usize = 100000; +/// Default value for max starlark stack size +pub(crate) const DEFAULT_STACK_SIZE: usize = 50; + /// Holds everything about an ongoing evaluation (local variables, globals, module resolution etc). pub struct Evaluator<'v, 'a> { // The module that is being used for this evaluation @@ -155,6 +162,8 @@ pub struct Evaluator<'v, 'a> { Option anyhow::Result>>>, /// Use in implementation of `print` function. pub(crate) print_handler: &'a (dyn PrintHandler + 'a), + /// Max size of starlark stack + pub(crate) max_callstack_size: Option, // The Starlark-level call-stack of functions. // Must go last because it's quite a big structure pub(crate) call_stack: CheapCallStack<'v>, @@ -226,6 +235,7 @@ impl<'v, 'a> Evaluator<'v, 'a> { print_handler: &StderrPrintHandler, verbose_gc: false, static_typechecking: false, + max_callstack_size: None, } } @@ -803,6 +813,19 @@ impl<'v, 'a> Evaluator<'v, 'a> { bc.run(self, &mut EvalCallbacksDisabled) } } + + /// Sets max call stack size. + /// Stack allocation will happen on entry point of evaluation if not allocated yet. + pub fn set_max_callstack_size(&mut self, stack_size: usize) -> anyhow::Result<()> { + if stack_size == 0 { + return Err(EvaluatorError::ZeroCallstackSize.into()); + } + if self.max_callstack_size.is_some() { + return Err(EvaluatorError::CallstackSizeAlreadySet.into()); + } + self.max_callstack_size = Some(stack_size); + Ok(()) + } } pub(crate) trait EvaluationCallbacks { diff --git a/starlark-rust/starlark/src/lib.rs b/starlark-rust/starlark/src/lib.rs index dc0006080ce12..06da0d43a969d 100644 --- a/starlark-rust/starlark/src/lib.rs +++ b/starlark-rust/starlark/src/lib.rs @@ -23,10 +23,12 @@ //! //! ``` //! # fn run() -> starlark::Result<()> { +//! use starlark::environment::Globals; +//! use starlark::environment::Module; //! use starlark::eval::Evaluator; -//! use starlark::environment::{Module, Globals}; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; //! use starlark::values::Value; -//! use starlark::syntax::{AstModule, Dialect}; //! //! let content = r#" //! def hello(): @@ -36,7 +38,8 @@ //! //! // We first parse the content, giving a filename and the Starlark //! // `Dialect` we'd like to use (we pick standard). -//! let ast: AstModule = AstModule::parse("hello_world.star", content.to_owned(), &Dialect::Standard)?; +//! let ast: AstModule = +//! AstModule::parse("hello_world.star", content.to_owned(), &Dialect::Standard)?; //! //! // We create a `Globals`, defining the standard library functions available. //! // The `standard` function uses those defined in the Starlark specification. @@ -108,13 +111,18 @@ //! #[macro_use] //! extern crate starlark; //! # fn run() -> starlark::Result<()> { -//! use starlark::environment::{GlobalsBuilder, Module}; -//! use starlark::eval::Evaluator; -//! use starlark::syntax::{AstModule, Dialect}; -//! use starlark::values::{none::NoneType, Value, ValueLike}; -//! use starlark::any::ProvidesStaticType; //! use std::cell::RefCell; //! +//! use starlark::any::ProvidesStaticType; +//! use starlark::environment::GlobalsBuilder; +//! use starlark::environment::Module; +//! use starlark::eval::Evaluator; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; +//! use starlark::values::none::NoneType; +//! use starlark::values::Value; +//! use starlark::values::ValueLike; +//! //! let content = r#" //! emit(1) //! emit(["test"]) @@ -127,7 +135,7 @@ //! //! impl Store { //! fn add(&self, x: String) { -//! self.0.borrow_mut().push(x) +//! self.0.borrow_mut().push(x) //! } //! } //! @@ -169,9 +177,12 @@ //! //! ``` //! # fn run() -> starlark::Result<()> { -//! use starlark::environment::{Globals, Module}; +//! use starlark::environment::Globals; +//! use starlark::environment::Module; //! use starlark::eval::Evaluator; -//! use starlark::syntax::{AstModule, Dialect, DialectTypes}; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; +//! use starlark::syntax::DialectTypes; //! //! let content = r#" //! def takes_int(x: int): @@ -180,7 +191,10 @@ //! "#; //! //! // Make the dialect enable types -//! let dialect = Dialect {enable_types: DialectTypes::Enable, ..Dialect::Standard}; +//! let dialect = Dialect { +//! enable_types: DialectTypes::Enable, +//! ..Dialect::Standard +//! }; //! // We could equally have done `dialect = Dialect::Extended`. //! let ast = AstModule::parse("json.star", content.to_owned(), &dialect)?; //! let globals = Globals::standard(); @@ -188,7 +202,11 @@ //! let mut eval = Evaluator::new(&module); //! let res = eval.eval_module(ast, &globals); //! // We expect this to fail, since it is a type violation -//! assert!(res.unwrap_err().to_string().contains("Value `test` of type `string` does not match the type annotation `int`")); +//! assert!( +//! res.unwrap_err() +//! .to_string() +//! .contains("Value `test` of type `string` does not match the type annotation `int`") +//! ); //! # Ok(()) //! # } //! # fn main(){ run().unwrap(); } @@ -202,16 +220,21 @@ //! //! ``` //! # fn run() -> starlark::Result<()> { -//! use starlark::environment::{FrozenModule, Globals, Module}; -//! use starlark::eval::{Evaluator, ReturnFileLoader}; -//! use starlark::syntax::{AstModule, Dialect}; +//! use starlark::environment::FrozenModule; +//! use starlark::environment::Globals; +//! use starlark::environment::Module; +//! use starlark::eval::Evaluator; +//! use starlark::eval::ReturnFileLoader; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; //! //! // Get the file contents (for the demo), in reality use `AstModule::parse_file`. //! fn get_source(file: &str) -> &str { //! match file { //! "a.star" => "a = 7", //! "b.star" => "b = 6", -//! _ => { r#" +//! _ => { +//! r#" //! load('a.star', 'a') //! load('b.star', 'b') //! ab = a * b @@ -221,27 +244,27 @@ //! } //! //! fn get_module(file: &str) -> starlark::Result { -//! let ast = AstModule::parse(file, get_source(file).to_owned(), &Dialect::Standard)?; -//! -//! // We can get the loaded modules from `ast.loads`. -//! // And ultimately produce a `loader` capable of giving those modules to Starlark. -//! let mut loads = Vec::new(); -//! for load in ast.loads() { -//! loads.push((load.module_id.to_owned(), get_module(load.module_id)?)); -//! } -//! let modules = loads.iter().map(|(a, b)| (a.as_str(), b)).collect(); -//! let mut loader = ReturnFileLoader { modules: &modules }; -//! -//! let globals = Globals::standard(); -//! let module = Module::new(); -//! { -//! let mut eval = Evaluator::new(&module); -//! eval.set_loader(&mut loader); -//! eval.eval_module(ast, &globals)?; -//! } -//! // After creating a module we freeze it, preventing further mutation. -//! // It can now be used as the input for other Starlark modules. -//! Ok(module.freeze()?) +//! let ast = AstModule::parse(file, get_source(file).to_owned(), &Dialect::Standard)?; +//! +//! // We can get the loaded modules from `ast.loads`. +//! // And ultimately produce a `loader` capable of giving those modules to Starlark. +//! let mut loads = Vec::new(); +//! for load in ast.loads() { +//! loads.push((load.module_id.to_owned(), get_module(load.module_id)?)); +//! } +//! let modules = loads.iter().map(|(a, b)| (a.as_str(), b)).collect(); +//! let mut loader = ReturnFileLoader { modules: &modules }; +//! +//! let globals = Globals::standard(); +//! let module = Module::new(); +//! { +//! let mut eval = Evaluator::new(&module); +//! eval.set_loader(&mut loader); +//! eval.eval_module(ast, &globals)?; +//! } +//! // After creating a module we freeze it, preventing further mutation. +//! // It can now be used as the input for other Starlark modules. +//! Ok(module.freeze()?) //! } //! //! let ab = get_module("ab.star")?; @@ -257,9 +280,11 @@ //! //! ``` //! # fn run() -> starlark::Result<()> { -//! use starlark::environment::{Globals, Module}; +//! use starlark::environment::Globals; +//! use starlark::environment::Module; //! use starlark::eval::Evaluator; -//! use starlark::syntax::{AstModule, Dialect}; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; //! use starlark::values::Value; //! //! let content = r#" @@ -292,13 +317,24 @@ //! //! ``` //! # fn run() -> starlark::Result<()> { -//! use starlark::environment::{Globals, Module}; +//! use std::fmt::Display; +//! use std::fmt::Write; +//! use std::fmt::{self}; +//! +//! use allocative::Allocative; +//! use starlark::environment::Globals; +//! use starlark::environment::Module; //! use starlark::eval::Evaluator; -//! use starlark::syntax::{AstModule, Dialect}; -//! use starlark::values::{Heap, StarlarkValue, Value, ValueError, ValueLike, ProvidesStaticType, NoSerialize}; //! use starlark::starlark_simple_value; -//! use std::fmt::{self, Display, Write}; -//! use allocative::Allocative; +//! use starlark::syntax::AstModule; +//! use starlark::syntax::Dialect; +//! use starlark::values::Heap; +//! use starlark::values::NoSerialize; +//! use starlark::values::ProvidesStaticType; +//! use starlark::values::StarlarkValue; +//! use starlark::values::Value; +//! use starlark::values::ValueError; +//! use starlark::values::ValueLike; //! use starlark_derive::starlark_value; //! //! // Define complex numbers @@ -318,8 +354,7 @@ //! #[starlark_value(type = "complex")] //! impl<'v> StarlarkValue<'v> for Complex { //! // How we add them -//! fn add(&self, rhs: Value<'v>, heap: &'v Heap) -//! -> Option>> { +//! fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option>> { //! if let Some(rhs) = rhs.downcast_ref::() { //! Some(Ok(heap.alloc(Complex { //! real: self.real + rhs.real, @@ -337,9 +372,15 @@ //! let globals = Globals::standard(); //! let module = Module::new(); //! // We inject some complex numbers into the module before we start. -//! let a = module.heap().alloc(Complex {real: 1, imaginary: 8}); +//! let a = module.heap().alloc(Complex { +//! real: 1, +//! imaginary: 8, +//! }); //! module.set("a", a); -//! let b = module.heap().alloc(Complex {real: 4, imaginary: 2}); +//! let b = module.heap().alloc(Complex { +//! real: 4, +//! imaginary: 2, +//! }); //! module.set("b", b); //! let mut eval = Evaluator::new(&module); //! let res = eval.eval_module(ast, &globals)?; diff --git a/starlark-rust/starlark/src/macros/mod.rs b/starlark-rust/starlark/src/macros/mod.rs index b5968a2d5477e..43b0f44c2d2dd 100644 --- a/starlark-rust/starlark/src/macros/mod.rs +++ b/starlark-rust/starlark/src/macros/mod.rs @@ -128,10 +128,14 @@ macro_rules! starlark_complex_values { /// Let's define a simple object, where `+x` makes the string uppercase: /// /// ``` -/// use starlark::values::{Heap, StarlarkValue, Value, ProvidesStaticType, NoSerialize}; -/// use starlark::{starlark_simple_value}; -/// use derive_more::Display; /// use allocative::Allocative; +/// use derive_more::Display; +/// use starlark::starlark_simple_value; +/// use starlark::values::Heap; +/// use starlark::values::NoSerialize; +/// use starlark::values::ProvidesStaticType; +/// use starlark::values::StarlarkValue; +/// use starlark::values::Value; /// use starlark_derive::starlark_value; /// /// #[derive(Debug, Display, ProvidesStaticType, NoSerialize, Allocative)] diff --git a/starlark-rust/starlark/src/stdlib/call_stack.rs b/starlark-rust/starlark/src/stdlib/call_stack.rs index 091c72ef0381e..254a482338105 100644 --- a/starlark-rust/starlark/src/stdlib/call_stack.rs +++ b/starlark-rust/starlark/src/stdlib/call_stack.rs @@ -17,11 +17,85 @@ //! Implementation of `call_stack` function. +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use allocative::Allocative; use starlark_derive::starlark_module; +use starlark_syntax::codemap::FileSpan; use crate as starlark; use crate::environment::GlobalsBuilder; +use crate::environment::Methods; +use crate::environment::MethodsBuilder; +use crate::environment::MethodsStatic; use crate::eval::Evaluator; +use crate::values::none::NoneOr; +use crate::values::starlark_value; +use crate::values::AllocValue; +use crate::values::Heap; +use crate::values::NoSerialize; +use crate::values::ProvidesStaticType; +use crate::values::StarlarkValue; +use crate::values::Trace; +use crate::values::Value; +use crate::StarlarkDocs; + +#[derive( + ProvidesStaticType, + Trace, + Allocative, + StarlarkDocs, + Debug, + NoSerialize, + Clone +)] +/// A frame of the call-stack. +struct StackFrame { + /// The name of the entry on the call-stack. + name: String, + /// The location of the definition, or [`None`] for native Rust functions. + location: Option, +} + +#[starlark_value(type = "StackFrame", StarlarkTypeRepr, UnpackValue)] +impl<'v> StarlarkValue<'v> for StackFrame { + fn get_methods() -> Option<&'static Methods> { + static RES: MethodsStatic = MethodsStatic::new(); + RES.methods(stack_frame_methods) + } +} + +impl<'v> AllocValue<'v> for StackFrame { + fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + heap.alloc_complex_no_freeze(self) + } +} + +impl Display for StackFrame { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "") + } +} + +#[starlark_module] +fn stack_frame_methods(builder: &mut MethodsBuilder) { + /// Returns the name of the entry on the call-stack. + #[starlark(attribute)] + fn func_name(this: &StackFrame) -> anyhow::Result { + Ok(this.name.clone()) + } + + /// Returns a path of the module from which the entry was called, or [`None`] for native Rust functions. + #[starlark(attribute)] + fn module_path(this: &StackFrame) -> anyhow::Result> { + match this.location { + Some(ref location) => Ok(Some(location.file.filename().to_owned())), + None => Ok(None), + } + } +} #[starlark_module] pub(crate) fn global(builder: &mut GlobalsBuilder) { @@ -43,6 +117,29 @@ pub(crate) fn global(builder: &mut GlobalsBuilder) { .truncate(stack.frames.len().saturating_sub(strip_frames as usize)); Ok(stack.to_string()) } + + /// Get a structural representation of the n-th call stack frame. + /// + /// With `n=0` returns `call_stack_frame` itself. + /// Returns `None` if `n` is greater than or equal to the stack size. + fn call_stack_frame( + #[starlark(require = pos)] n: u32, + eval: &mut Evaluator, + ) -> anyhow::Result> { + let stack = eval.call_stack(); + let n = n as usize; + if n >= stack.frames.len() { + return Ok(NoneOr::None); + } + match stack.frames.get(stack.frames.len() - n - 1) { + Some(frame) => Ok(NoneOr::Other(StackFrame { + name: frame.name.clone(), + location: frame.location.clone(), + })), + + None => Ok(NoneOr::None), + } + } } #[cfg(test)] @@ -107,6 +204,32 @@ def bar(): s = call_stack(strip_frames=10) return not bool(s) +foo() + "#, + ); + } + + #[test] + fn test_call_stack_frame() { + let mut a = Assert::new(); + a.globals_add(global); + a.is_true( + r#" +def foo(): + return bar() + +def bar(): + return all([ + "call_stack_frame" == call_stack_frame(0).func_name, + "assert.bzl" == call_stack_frame(0).module_path, + "bar" == call_stack_frame(1).func_name, + "assert.bzl" == call_stack_frame(1).module_path, + "foo" == call_stack_frame(2).func_name, + "assert.bzl" == call_stack_frame(2).module_path, + None == call_stack_frame(3), + None == call_stack_frame(4), + ]) + foo() "#, ); diff --git a/starlark-rust/starlark/src/stdlib/funcs/list.rs b/starlark-rust/starlark/src/stdlib/funcs/list.rs index 1aada74975489..6bbf132fa5edf 100644 --- a/starlark-rust/starlark/src/stdlib/funcs/list.rs +++ b/starlark-rust/starlark/src/stdlib/funcs/list.rs @@ -63,7 +63,7 @@ impl TyCustomFunctionImpl for ListType { oracle.validate_fn_call(span, &LIST, args)?; - if let Some(arg) = args.get(0) { + if let Some(arg) = args.first() { // This is infallible after the check above. if let Arg::Pos(arg_ty) = &arg.node { // This is also infallible. diff --git a/starlark-rust/starlark/src/stdlib/funcs/other.rs b/starlark-rust/starlark/src/stdlib/funcs/other.rs index 419404f8f893e..68fc9eb056cc8 100644 --- a/starlark-rust/starlark/src/stdlib/funcs/other.rs +++ b/starlark-rust/starlark/src/stdlib/funcs/other.rs @@ -476,7 +476,7 @@ pub(crate) fn register_other(builder: &mut GlobalsBuilder) { >, base: Option, heap: &'v Heap, - ) -> anyhow::Result> { + ) -> starlark::Result> { let Some(a) = a else { return Ok(ValueOfUnchecked::new(heap.alloc(0))); }; @@ -488,7 +488,8 @@ pub(crate) fn register_other(builder: &mut GlobalsBuilder) { return Err(anyhow::anyhow!( "{} is not a valid base, int() base must be >= 2 and <= 36", base - )); + ) + .into()); } let (negate, s) = { match s.chars().next() { @@ -533,7 +534,7 @@ pub(crate) fn register_other(builder: &mut GlobalsBuilder) { }; // We already handled the sign above, so we are not trying to parse another sign. if s.starts_with('-') || s.starts_with('+') { - return Err(anyhow::anyhow!("Cannot parse `{}` as an integer", s,)); + return Err(anyhow::anyhow!("Cannot parse `{}` as an integer", s,).into()); } let x = StarlarkInt::from_str_radix(s, base)?; @@ -546,7 +547,8 @@ pub(crate) fn register_other(builder: &mut GlobalsBuilder) { return Err(anyhow::anyhow!( "int() cannot convert non-string with explicit base '{}'", base - )); + ) + .into()); } match num_or_bool { diff --git a/starlark-rust/starlark/src/stdlib/json.rs b/starlark-rust/starlark/src/stdlib/json.rs index c844f25608561..75b412aafd33a 100644 --- a/starlark-rust/starlark/src/stdlib/json.rs +++ b/starlark-rust/starlark/src/stdlib/json.rs @@ -68,6 +68,8 @@ impl<'v, 'a> AllocValue<'v> for &'a serde_json::Number { impl<'v> AllocValue<'v> for serde_json::Number { fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } @@ -90,6 +92,8 @@ impl<'a> AllocFrozenValue for &'a serde_json::Number { impl AllocFrozenValue for serde_json::Number { fn alloc_frozen_value(self, heap: &FrozenHeap) -> FrozenValue { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } @@ -116,6 +120,8 @@ impl<'a, 'v> AllocValue<'v> for &'a serde_json::Map { impl<'v> AllocValue<'v> for serde_json::Map { fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } @@ -128,6 +134,8 @@ impl<'a> AllocFrozenValue for &'a serde_json::Map { impl AllocFrozenValue for serde_json::Map { fn alloc_frozen_value(self, heap: &FrozenHeap) -> FrozenValue { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } @@ -161,6 +169,8 @@ impl<'v, 'a> AllocValue<'v> for &'a serde_json::Value { impl<'v> AllocValue<'v> for serde_json::Value { fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } @@ -180,6 +190,8 @@ impl<'a> AllocFrozenValue for &'a serde_json::Value { impl AllocFrozenValue for serde_json::Value { fn alloc_frozen_value(self, heap: &FrozenHeap) -> FrozenValue { + // If you follow this hint, it becomes infinite recursion + #[allow(clippy::needless_borrows_for_generic_args)] heap.alloc(&self) } } diff --git a/starlark-rust/starlark/src/stdlib/string.rs b/starlark-rust/starlark/src/stdlib/string.rs index a6516cd2d3f7a..5d0aeffd32dd6 100644 --- a/starlark-rust/starlark/src/stdlib/string.rs +++ b/starlark-rust/starlark/src/stdlib/string.rs @@ -143,8 +143,7 @@ pub(crate) fn string_methods(builder: &mut MethodsBuilder) { /// /// ``` /// # starlark::assert::is_true(r#" - /// list("Hello, 世界".elems()) == [ - /// "H", "e", "l", "l", "o", ",", " ", "世", "界"] + /// list("Hello, 世界".elems()) == ["H", "e", "l", "l", "o", ",", " ", "世", "界"] /// # "#); /// ``` fn elems<'v>( diff --git a/starlark-rust/starlark/src/typing/user.rs b/starlark-rust/starlark/src/typing/user.rs index 2c007bab515bf..40d317068d57c 100644 --- a/starlark-rust/starlark/src/typing/user.rs +++ b/starlark-rust/starlark/src/typing/user.rs @@ -156,20 +156,14 @@ impl TyUser { iter_item, _non_exhaustive: (), } = params; - if callable.is_some() { - if !base.is_callable() { - return Err(TyUserError::CallableNotCallable(name).into()); - } + if callable.is_some() && !base.is_callable() { + return Err(TyUserError::CallableNotCallable(name).into()); } - if index.is_some() { - if !base.is_indexable() { - return Err(TyUserError::IndexableNotIndexable(name).into()); - } + if index.is_some() && !base.is_indexable() { + return Err(TyUserError::IndexableNotIndexable(name).into()); } - if iter_item.is_some() { - if base.iter_item().is_err() { - return Err(TyUserError::IterableNotIterable(name).into()); - } + if iter_item.is_some() && base.iter_item().is_err() { + return Err(TyUserError::IterableNotIterable(name).into()); } Ok(TyUser { name, diff --git a/starlark-rust/starlark/src/values/frozen_ref.rs b/starlark-rust/starlark/src/values/frozen_ref.rs index 2c144218f8482..a09ccb6f2eb98 100644 --- a/starlark-rust/starlark/src/values/frozen_ref.rs +++ b/starlark-rust/starlark/src/values/frozen_ref.rs @@ -29,10 +29,12 @@ use std::sync::atomic; use allocative::Allocative; use dupe::Clone_; use dupe::Copy_; +use dupe::Dupe; use dupe::Dupe_; use crate::values::Freeze; use crate::values::Freezer; +use crate::values::FrozenHeapRef; use crate::values::Trace; use crate::values::Tracer; @@ -78,6 +80,26 @@ impl<'f, T: 'f + ?Sized> FrozenRef<'f, T> { value: f(self.value), } } + + /// Fallible map the reference to another one. + pub fn try_map_result(self, f: F) -> Result, E> + where + for<'v> F: FnOnce(&'v T) -> Result<&'v U, E>, + { + Ok(FrozenRef { + value: f(self.value)?, + }) + } + + /// Optionally map the reference to another one. + pub fn try_map_option(self, f: F) -> Option> + where + for<'v> F: FnOnce(&'v T) -> Option<&'v U>, + { + Some(FrozenRef { + value: f(self.value)?, + }) + } } impl<'f, T: ?Sized + Display> Display for FrozenRef<'f, T> { @@ -186,3 +208,92 @@ impl AtomicFrozenRefOption { ); } } + +/// Same as a `FrozenRef`, but it keeps itself alive by storing a reference to the owning heap. +/// +/// Usually constructed from an `OwnedFrozenValueTyped`. +#[derive(Clone, Dupe, Allocative)] +pub struct OwnedFrozenRef { + owner: FrozenHeapRef, + // Invariant: this FrozenValue must be kept alive by the `owner` field. + value: FrozenRef<'static, T>, +} + +impl OwnedFrozenRef { + /// Creates a new `OwnedFrozenRef` pointing at the given value. + /// + /// ## Safety + /// + /// The reference must be kept alive by the owning heap + pub unsafe fn new_unchecked(value: &'static T, owner: FrozenHeapRef) -> OwnedFrozenRef { + OwnedFrozenRef { + owner, + value: FrozenRef::new(value), + } + } + + /// Returns a reference to the underlying value. + pub fn as_ref<'a>(&'a self) -> &'a T { + self.value.as_ref() + } + + /// Converts `self` into a new reference that points at something reachable from the previous. + /// + /// See the caveats on `[starlark::values::OwnedFrozenValue::map]` + pub fn map(self, f: F) -> OwnedFrozenRef + where + for<'v> F: FnOnce(&'v T) -> &'v U, + { + OwnedFrozenRef { + owner: self.owner, + value: self.value.map(f), + } + } + + /// Fallible map the reference to another one. + pub fn try_map_result(self, f: F) -> Result, E> + where + for<'v> F: FnOnce(&'v T) -> Result<&'v U, E>, + { + Ok(OwnedFrozenRef { + owner: self.owner, + value: self.value.try_map_result(f)?, + }) + } + + /// Optionally map the reference to another one. + pub fn try_map_option(self, f: F) -> Option> + where + for<'v> F: FnOnce(&'v T) -> Option<&'v U>, + { + Some(OwnedFrozenRef { + owner: self.owner, + value: self.value.try_map_option(f)?, + }) + } + + /// Get a reference to the owning frozen heap + pub fn owner(&self) -> &FrozenHeapRef { + &self.owner + } +} + +impl fmt::Debug for OwnedFrozenRef { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.value, f) + } +} + +impl fmt::Display for OwnedFrozenRef { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.value, f) + } +} + +impl Deref for OwnedFrozenRef { + type Target = T; + + fn deref(&self) -> &T { + self.as_ref() + } +} diff --git a/starlark-rust/starlark/src/values/layout/typed/string.rs b/starlark-rust/starlark/src/values/layout/typed/string.rs index 4f15bd459569d..0392e9ee6167e 100644 --- a/starlark-rust/starlark/src/values/layout/typed/string.rs +++ b/starlark-rust/starlark/src/values/layout/typed/string.rs @@ -49,7 +49,8 @@ use crate::values::ValueTyped; /// /// ``` /// use starlark::const_frozen_string; -/// use starlark::values::{FrozenStringValue, FrozenValue}; +/// use starlark::values::FrozenStringValue; +/// use starlark::values::FrozenValue; /// /// let fv: FrozenStringValue = const_frozen_string!("magic"); /// assert_eq!("magic", fv.as_str()); diff --git a/starlark-rust/starlark/src/values/mod.rs b/starlark-rust/starlark/src/values/mod.rs index ee6010931b8de..72e7bb93679a0 100644 --- a/starlark-rust/starlark/src/values/mod.rs +++ b/starlark-rust/starlark/src/values/mod.rs @@ -47,6 +47,7 @@ pub use crate::values::demand::Demand; pub use crate::values::error::ValueError; pub use crate::values::freeze::Freeze; pub use crate::values::frozen_ref::FrozenRef; +pub use crate::values::frozen_ref::OwnedFrozenRef; pub use crate::values::iter::StarlarkIterator; pub use crate::values::layout::complex::ValueTypedComplex; pub use crate::values::layout::heap::heap_type::Freezer; diff --git a/starlark-rust/starlark/src/values/owned.rs b/starlark-rust/starlark/src/values/owned.rs index 138f36bac3095..842d82d89c40d 100644 --- a/starlark-rust/starlark/src/values/owned.rs +++ b/starlark-rust/starlark/src/values/owned.rs @@ -32,6 +32,7 @@ use crate::values::FrozenHeap; use crate::values::FrozenHeapRef; use crate::values::FrozenValue; use crate::values::FrozenValueTyped; +use crate::values::OwnedFrozenRef; use crate::values::StarlarkValue; use crate::values::Value; @@ -89,7 +90,8 @@ impl OwnedFrozenValue { /// `owner`, typically because the value was created on the heap. /// /// ``` - /// use starlark::values::{FrozenHeap, OwnedFrozenValue}; + /// use starlark::values::FrozenHeap; + /// use starlark::values::OwnedFrozenValue; /// let heap = FrozenHeap::new(); /// let value = heap.alloc("test"); /// unsafe { OwnedFrozenValue::new(heap.into_ref(), value) }; @@ -243,6 +245,12 @@ impl> OwnedFrozenValueTyped { } } + /// Convert to an owned ref. + pub fn into_owned_frozen_ref(self) -> OwnedFrozenRef { + // SAFETY: Heap matches the value + unsafe { OwnedFrozenRef::new_unchecked(self.value.as_ref(), self.owner) } + } + /// Obtain a reference to the FrozenHeap that owns this value. pub fn owner(&self) -> &FrozenHeapRef { &self.owner diff --git a/starlark-rust/starlark/src/values/trace.rs b/starlark-rust/starlark/src/values/trace.rs index 44dfd8e7bc4ce..6c7d9083d7940 100644 --- a/starlark-rust/starlark/src/values/trace.rs +++ b/starlark-rust/starlark/src/values/trace.rs @@ -37,6 +37,7 @@ use std::sync::Mutex; use either::Either; use hashbrown::raw::RawTable; use starlark_map::small_set::SmallSet; +use starlark_map::Hashed; use crate::collections::SmallMap; use crate::values::FrozenValue; @@ -54,7 +55,7 @@ use crate::values::Value; /// /// #[derive(Trace)] /// struct MySet<'v> { -/// keys: Vec> +/// keys: Vec>, /// } /// ``` pub unsafe trait Trace<'v> { @@ -104,6 +105,12 @@ unsafe impl<'v, T: Trace<'v>> Trace<'v> for SmallSet { } } +unsafe impl<'v, T: Trace<'v>> Trace<'v> for Hashed { + fn trace(&mut self, tracer: &Tracer<'v>) { + self.key_mut().trace(tracer); + } +} + unsafe impl<'v, T: Trace<'v>> Trace<'v> for Option { fn trace(&mut self, tracer: &Tracer<'v>) { if let Some(x) = self { diff --git a/starlark-rust/starlark/src/values/traits.rs b/starlark-rust/starlark/src/values/traits.rs index a892529335451..eb96123e57f2b 100644 --- a/starlark-rust/starlark/src/values/traits.rs +++ b/starlark-rust/starlark/src/values/traits.rs @@ -76,20 +76,40 @@ use crate::values::ValueError; /// generate `One` and `FrozenOne` aliases. /// /// ``` -/// use starlark::values::{ProvidesStaticType, ComplexValue, Coerce, Freezer, FrozenValue, StarlarkValue, Value, ValueLike, Trace, Tracer, Freeze, NoSerialize}; -/// use starlark::{starlark_complex_value}; -/// use derive_more::Display; /// use allocative::Allocative; +/// use derive_more::Display; +/// use starlark::starlark_complex_value; +/// use starlark::values::Coerce; +/// use starlark::values::ComplexValue; +/// use starlark::values::Freeze; +/// use starlark::values::Freezer; +/// use starlark::values::FrozenValue; +/// use starlark::values::NoSerialize; +/// use starlark::values::ProvidesStaticType; +/// use starlark::values::StarlarkValue; +/// use starlark::values::Trace; +/// use starlark::values::Tracer; +/// use starlark::values::Value; +/// use starlark::values::ValueLike; /// use starlark_derive::starlark_value; /// -/// #[derive(Debug, Trace, Coerce, Display, ProvidesStaticType, NoSerialize, Allocative)] +/// #[derive( +/// Debug, +/// Trace, +/// Coerce, +/// Display, +/// ProvidesStaticType, +/// NoSerialize, +/// Allocative +/// )] /// #[repr(C)] /// struct OneGen(V); /// starlark_complex_value!(One); /// /// #[starlark_value(type = "one")] /// impl<'v, V: ValueLike<'v> + 'v> StarlarkValue<'v> for OneGen -/// where Self: ProvidesStaticType<'v>, +/// where +/// Self: ProvidesStaticType<'v>, /// { /// // To implement methods which work for both `One` and `FrozenOne`, /// // use the `ValueLike` trait. @@ -187,12 +207,12 @@ where /// proc macro: /// /// ``` -/// use starlark::values::StarlarkValue; -/// use starlark::values::ProvidesStaticType; -/// use starlark::values::NoSerialize; +/// use allocative::Allocative; /// # use starlark::starlark_simple_value; /// use derive_more::Display; -/// use allocative::Allocative; +/// use starlark::values::NoSerialize; +/// use starlark::values::ProvidesStaticType; +/// use starlark::values::StarlarkValue; /// use starlark_derive::starlark_value; /// /// #[derive(Debug, Display, ProvidesStaticType, NoSerialize, Allocative)] @@ -200,8 +220,7 @@ where /// struct Foo; /// # starlark_simple_value!(Foo); /// #[starlark_value(type = "foo")] -/// impl<'v> StarlarkValue<'v> for Foo { -/// } +/// impl<'v> StarlarkValue<'v> for Foo {} /// ``` /// /// Every additional field enables further features in Starlark. In most cases the default diff --git a/starlark-rust/starlark/src/values/types/any.rs b/starlark-rust/starlark/src/values/types/any.rs index 3d3f37e5bd3d4..d66afe325e14a 100644 --- a/starlark-rust/starlark/src/values/types/any.rs +++ b/starlark-rust/starlark/src/values/types/any.rs @@ -28,14 +28,15 @@ //! #[macro_use] //! extern crate starlark; //! # fn main() { -//! use starlark::assert::Assert; -//! use starlark::environment::GlobalsBuilder; -//! use starlark::values::Value; -//! use starlark::values::any::StarlarkAny; //! use std::fmt; //! use std::fmt::Display; //! use std::time::Instant; //! +//! use starlark::assert::Assert; +//! use starlark::environment::GlobalsBuilder; +//! use starlark::values::any::StarlarkAny; +//! use starlark::values::Value; +//! //! #[derive(Debug)] //! struct MyInstant(Instant); //! @@ -52,19 +53,26 @@ //! } //! //! fn elapsed(x: Value) -> anyhow::Result { -//! Ok(StarlarkAny::::get(x).unwrap().0.elapsed().as_secs_f64().to_string()) +//! Ok(StarlarkAny::::get(x) +//! .unwrap() +//! .0 +//! .elapsed() +//! .as_secs_f64() +//! .to_string()) //! } //! } //! //! let mut a = Assert::new(); //! a.globals_add(globals); -//! a.pass(r#" +//! a.pass( +//! r#" //! duration = start() //! y = 100 //! for x in range(100): //! y += x //! print(elapsed(duration)) -//! "#); +//! "#, +//! ); //! # } //! ``` diff --git a/starlark-rust/starlark/src/values/types/exported_name.rs b/starlark-rust/starlark/src/values/types/exported_name.rs index 7357c93e9d324..f70a33f0cb910 100644 --- a/starlark-rust/starlark/src/values/types/exported_name.rs +++ b/starlark-rust/starlark/src/values/types/exported_name.rs @@ -63,24 +63,37 @@ impl<'a> Eq for BorrowedExportedName<'a> {} /// ``` /// use allocative::Allocative; /// use starlark::eval::Evaluator; -/// use starlark::values::exported_name::{ExportedName, FrozenExportedName}; +/// use starlark::values::exported_name::ExportedName; +/// use starlark::values::exported_name::FrozenExportedName; /// use starlark::values::StarlarkValue; -/// use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value}; +/// use starlark_derive::starlark_value; +/// use starlark_derive::NoSerialize; +/// use starlark_derive::ProvidesStaticType; /// -/// #[derive(Debug, NoSerialize, ProvidesStaticType, Allocative, derive_more::Display)] +/// #[derive( +/// Debug, +/// NoSerialize, +/// ProvidesStaticType, +/// Allocative, +/// derive_more::Display +/// )] /// #[display(fmt = "{:?}", "self")] /// struct MyStruct { -/// name: T, +/// name: T, /// } /// /// #[starlark_value(type = "MyStruct")] /// impl<'v, T: ExportedName> StarlarkValue<'v> for MyStruct { -/// type Canonical = MyStruct; +/// type Canonical = MyStruct; /// -/// fn export_as(&self, variable_name: &str, _eval: &mut Evaluator<'v, '_>) -> starlark::Result<()> { -/// self.name.try_export_as(variable_name); -/// Ok(()) -/// } +/// fn export_as( +/// &self, +/// variable_name: &str, +/// _eval: &mut Evaluator<'v, '_>, +/// ) -> starlark::Result<()> { +/// self.name.try_export_as(variable_name); +/// Ok(()) +/// } /// } /// ``` /// diff --git a/starlark-rust/starlark/src/values/types/int_or_big.rs b/starlark-rust/starlark/src/values/types/int_or_big.rs index 17ef69d2e6730..44c4c53b85a58 100644 --- a/starlark-rust/starlark/src/values/types/int_or_big.rs +++ b/starlark-rust/starlark/src/values/types/int_or_big.rs @@ -87,7 +87,7 @@ impl FromStr for StarlarkInt { } impl StarlarkInt { - pub(crate) fn from_str_radix(s: &str, base: u32) -> anyhow::Result { + pub(crate) fn from_str_radix(s: &str, base: u32) -> crate::Result { Ok(StarlarkInt::from(TokenInt::from_str_radix(s, base)?)) } diff --git a/starlark-rust/starlark/src/values/types/starlark_value_as_type.rs b/starlark-rust/starlark/src/values/types/starlark_value_as_type.rs index 06526f79b91a8..53a97216cf0a2 100644 --- a/starlark-rust/starlark/src/values/types/starlark_value_as_type.rs +++ b/starlark-rust/starlark/src/values/types/starlark_value_as_type.rs @@ -65,17 +65,23 @@ impl Display for StarlarkValueAsTypeStarlarkValue { /// /// ``` /// use allocative::Allocative; +/// use starlark::any::ProvidesStaticType; /// use starlark::environment::GlobalsBuilder; +/// use starlark::values::starlark_value; /// use starlark::values::starlark_value_as_type::StarlarkValueAsType; +/// use starlark::values::NoSerialize; /// use starlark::values::StarlarkValue; -/// use starlark::any::ProvidesStaticType; -/// use starlark::values::{NoSerialize, starlark_value}; -/// #[derive(Debug, derive_more::Display, Allocative, ProvidesStaticType, NoSerialize)] +/// #[derive( +/// Debug, +/// derive_more::Display, +/// Allocative, +/// ProvidesStaticType, +/// NoSerialize +/// )] /// struct Temperature; /// /// #[starlark_value(type = "temperature")] -/// impl<'v> StarlarkValue<'v> for Temperature { -/// } +/// impl<'v> StarlarkValue<'v> for Temperature {} /// /// fn my_type_globals(globals: &mut GlobalsBuilder) { /// // This can now be used like: diff --git a/starlark-rust/starlark/src/values/types/string/intern/interner.rs b/starlark-rust/starlark/src/values/types/string/intern/interner.rs index 0e2d50d209200..1fbc996a0b8c5 100644 --- a/starlark-rust/starlark/src/values/types/string/intern/interner.rs +++ b/starlark-rust/starlark/src/values/types/string/intern/interner.rs @@ -51,7 +51,7 @@ impl FrozenStringInterner { } #[cfg(test)] -mod test { +mod tests { use crate::collections::Hashed; use crate::values::string::intern::interner::FrozenStringInterner; use crate::values::FrozenHeap; diff --git a/starlark-rust/starlark/src/values/unpack.rs b/starlark-rust/starlark/src/values/unpack.rs index d16062cc02bb6..e7c6ce005dbfe 100644 --- a/starlark-rust/starlark/src/values/unpack.rs +++ b/starlark-rust/starlark/src/values/unpack.rs @@ -40,7 +40,13 @@ use crate::values::ValueError; /// # use starlark::any::ProvidesStaticType; /// # use starlark::values::{NoSerialize, StarlarkValue, starlark_value}; /// -/// #[derive(Debug, derive_more::Display, Allocative, NoSerialize, ProvidesStaticType)] +/// #[derive( +/// Debug, +/// derive_more::Display, +/// Allocative, +/// NoSerialize, +/// ProvidesStaticType +/// )] /// struct MySimpleValue; /// /// #[starlark_value(type = "MySimpleValue", UnpackValue, StarlarkTypeRepr)] diff --git a/starlark-rust/starlark/testcases/eval/go/README.md b/starlark-rust/starlark/testcases/eval/go/README.md index 4eab164d768c5..f078fc4f3e227 100644 --- a/starlark-rust/starlark/testcases/eval/go/README.md +++ b/starlark-rust/starlark/testcases/eval/go/README.md @@ -1,5 +1,7 @@ # Go evaluation test cases -The Go Starlark project maintains a set of test cases, which were mirrored here. The original source -is https://github.com/google/starlark-go/blob/e81fc95f7bd5bb1495fe69f27c1a99fcc77caa48/starlark/testdata/. -Note that some files were not copied, because they are unsuitable tests for Starlark, as described in the `test_go` function. +The Go Starlark project maintains a set of test cases, which were mirrored here. +The original source is +https://github.com/google/starlark-go/blob/e81fc95f7bd5bb1495fe69f27c1a99fcc77caa48/starlark/testdata/. +Note that some files were not copied, because they are unsuitable tests for +Starlark, as described in the `test_go` function. diff --git a/starlark-rust/starlark_bin/BUCK b/starlark-rust/starlark_bin/BUCK index 2fd120c1dc0fe..11a68c127fbfa 100644 --- a/starlark-rust/starlark_bin/BUCK +++ b/starlark-rust/starlark_bin/BUCK @@ -8,6 +8,7 @@ buck_rust_binary( ["bin/**/*.rs"], ), crate_root = "bin/main.rs", + env = {"CARGO_PKG_VERSION": "0.0"}, # So our OSS builds can support --version deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:argfile", diff --git a/starlark-rust/starlark_bin/Cargo.toml b/starlark-rust/starlark_bin/Cargo.toml index 09aa8daed2d70..ca48a8afd2041 100644 --- a/starlark-rust/starlark_bin/Cargo.toml +++ b/starlark-rust/starlark_bin/Cargo.toml @@ -11,14 +11,14 @@ keywords = ["starlark", "skylark", "language", "interpreter"] license = "Apache-2.0" name = "starlark_bin" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [dependencies] dupe = { workspace = true } -starlark = { version = "0.10.0", path = "../starlark" } -starlark_lsp = { version = "0.10.0", path = "../starlark_lsp" } -starlark_map = { version = "0.10.0", path = "../starlark_map" } +starlark = { version = "0.12.0", path = "../starlark" } +starlark_lsp = { version = "0.12.0", path = "../starlark_lsp" } +starlark_map = { version = "0.12.0", path = "../starlark_map" } anyhow = "1.0.65" argfile = "0.1.0" diff --git a/starlark-rust/starlark_bin/bin/bazel.rs b/starlark-rust/starlark_bin/bin/bazel.rs index c057789108f8b..566e7e60de4a9 100644 --- a/starlark-rust/starlark_bin/bin/bazel.rs +++ b/starlark-rust/starlark_bin/bin/bazel.rs @@ -23,6 +23,8 @@ //! interface develops. After the API of the `LspContext` trait stabilizes, this //! module will be removed, and extracted to its own project. +mod label; + use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; @@ -56,6 +58,7 @@ use starlark_lsp::server::LspEvalResult; use starlark_lsp::server::LspUrl; use starlark_lsp::server::StringLiteralResult; +use self::label::Label; use crate::eval::dialect; use crate::eval::globals; use crate::eval::ContextMode; @@ -76,21 +79,18 @@ enum ContextError { enum ResolveLoadError { /// Attempted to resolve a relative path, but no current_file_path was provided, /// so it is not known what to resolve the path against. - #[error("Relative path `{}` provided, but current_file_path could not be determined", .0)] - MissingCurrentFilePath(String), + #[error("Relative label `{}` provided, but current_file_path could not be determined", .0)] + MissingCurrentFilePath(Label), /// The scheme provided was not correct or supported. #[error("Url `{}` was expected to be of type `{}`", .1, .0)] WrongScheme(String, LspUrl), /// Received a load for an absolute path from the root of the workspace, but the /// path to the workspace root was not provided. - #[error("Path `//{}` is absolute from the root of the workspace, but no workspace root was provided", .0)] - MissingWorkspaceRoot(String), - /// Unable to parse the given path. - #[error("Unable to parse the load path `{}`", .0)] - CannotParsePath(String), + #[error("Label `{}` is absolute from the root of the workspace, but no workspace root was provided", .0)] + MissingWorkspaceRoot(Label), /// The path contained a repository name that is not known to Bazel. - #[error("Cannot resolve path `{}` because the repository `{}` is unknown", .0, .1)] - UnknownRepository(String, String), + #[error("Cannot resolve label `{}` because the repository `{}` is unknown", .0, .1)] + UnknownRepository(Label, String), /// The path contained a target name that does not resolve to an existing file. #[error("Cannot resolve path `{}` because the file does not exist", .0)] TargetNotFound(String), @@ -393,94 +393,74 @@ impl BazelContext { .map(|external_output_base| external_output_base.join(repository_name)) } - fn resolve_folder<'a>( + /// Finds the directory that is the root of a package, given a label + fn resolve_folder( &self, - path: &'a str, + label: &Label, current_file: &LspUrl, workspace_root: Option<&Path>, - resolved_filename: &mut Option<&'a str>, ) -> anyhow::Result { - let original_path = path; - if let Some((repository, path)) = path.split_once("//") { - // The repository may be prefixed with an '@', but it's optional in Buck2. - let repository = if let Some(without_at) = repository.strip_prefix('@') { - without_at - } else { - repository - }; - - // Find the root we're resolving from. There's quite a few cases to consider here: - // - `repository` is empty, and we're resolving from the workspace root. - // - `repository` is empty, and we're resolving from a known remote repository. - // - `repository` is not empty, and refers to the current repository (the workspace). - // - `repository` is not empty, and refers to a known remote repository. - // - // Also with all of these cases, we need to consider if we have build system - // information or not. If not, we can't resolve any remote repositories, and we can't - // know whether a repository name refers to the workspace or not. - let resolve_root = match (repository, current_file) { - // Repository is empty, and we know what file we're resolving from. Use the build - // system information to check if we're in a known remote repository, and what the - // root is. Fall back to the `workspace_root` otherwise. - ("", LspUrl::File(current_file)) => { - if let Some((_, remote_repository_root)) = - self.get_repository_for_path(current_file) - { - Some(Cow::Borrowed(remote_repository_root)) - } else { - workspace_root.map(Cow::Borrowed) - } + // Find the root we're resolving from. There's quite a few cases to consider here: + // - `repository` is empty, and we're resolving from the workspace root. + // - `repository` is empty, and we're resolving from a known remote repository. + // - `repository` is not empty, and refers to the current repository (the workspace). + // - `repository` is not empty, and refers to a known remote repository. + // + // Also with all of these cases, we need to consider if we have build system + // information or not. If not, we can't resolve any remote repositories, and we can't + // know whether a repository name refers to the workspace or not. + let resolve_root = match (&label.repo, current_file) { + // Repository is empty, and we know what file we're resolving from. Use the build + // system information to check if we're in a known remote repository, and what the + // root is. Fall back to the `workspace_root` otherwise. + (None, LspUrl::File(current_file)) => { + if let Some((_, remote_repository_root)) = + self.get_repository_for_path(current_file) + { + Some(Cow::Borrowed(remote_repository_root)) + } else { + workspace_root.map(Cow::Borrowed) } - // No repository in the load path, and we don't have build system information, or - // an `LspUrl` we can't use to check the root. Use the workspace root. - ("", _) => workspace_root.map(Cow::Borrowed), - // We have a repository name and build system information. Check if the repository - // name refers to the workspace, and if so, use the workspace root. If not, check - // if it refers to a known remote repository, and if so, use that root. - // Otherwise, fail with an error. - (repository, _) => { - if matches!(self.workspace_name.as_ref(), Some(name) if name == repository) { - workspace_root.map(Cow::Borrowed) - } else if let Some(remote_repository_root) = - self.get_repository_path(repository).map(Cow::Owned) - { - Some(remote_repository_root) - } else { - return Err(ResolveLoadError::UnknownRepository( - original_path.to_owned(), - repository.to_owned(), - ) - .into()); - } + } + // No repository in the load path, and we don't have build system information, or + // an `LspUrl` we can't use to check the root. Use the workspace root. + (None, _) => workspace_root.map(Cow::Borrowed), + // We have a repository name and build system information. Check if the repository + // name refers to the workspace, and if so, use the workspace root. If not, check + // if it refers to a known remote repository, and if so, use that root. + // Otherwise, fail with an error. + (Some(repository), _) => { + if matches!(self.workspace_name.as_ref(), Some(name) if name == &repository.name) { + workspace_root.map(Cow::Borrowed) + } else if let Some(remote_repository_root) = + self.get_repository_path(&repository.name).map(Cow::Owned) + { + Some(remote_repository_root) + } else { + return Err(ResolveLoadError::UnknownRepository( + label.clone(), + repository.name.clone(), + ) + .into()); } - }; + } + }; + if let Some(package) = &label.package { // Resolve from the root of the repository. - match (path.split_once(':'), resolve_root) { - (Some((subfolder, filename)), Some(resolve_root)) => { - resolved_filename.replace(filename); - Ok(resolve_root.join(subfolder)) - } - (None, Some(resolve_root)) => Ok(resolve_root.join(path)), - (Some(_), None) => { - Err(ResolveLoadError::MissingWorkspaceRoot(original_path.to_owned()).into()) - } - (None, _) => { - Err(ResolveLoadError::CannotParsePath(original_path.to_owned()).into()) - } + match resolve_root { + Some(resolve_root) => Ok(resolve_root.join(package)), + None => Err(ResolveLoadError::MissingWorkspaceRoot(label.clone()).into()), } - } else if let Some((folder, filename)) = path.split_once(':') { - resolved_filename.replace(filename); - - // Resolve relative paths from the current file. + } else { + // If we don't have a package, this is relative to the current file, + // so resolve relative paths from the current file. match current_file { LspUrl::File(current_file_path) => { let current_file_dir = current_file_path.parent(); match current_file_dir { - Some(current_file_dir) => Ok(current_file_dir.join(folder)), - None => { - Err(ResolveLoadError::MissingCurrentFilePath(path.to_owned()).into()) - } + Some(current_file_dir) => Ok(current_file_dir.to_owned()), + None => Err(ResolveLoadError::MissingCurrentFilePath(label.clone()).into()), } } _ => Err( @@ -488,8 +468,6 @@ impl BazelContext { .into(), ), } - } else { - Err(ResolveLoadError::CannotParsePath(path.to_owned()).into()) } } @@ -528,10 +506,13 @@ impl BazelContext { // Find the actual folder on disk we're looking at. let (from_path, render_base) = match from { FilesystemCompletionRoot::Path(path) => (path.to_owned(), path.to_string_lossy()), - FilesystemCompletionRoot::String(str) => ( - self.resolve_folder(str, current_file, workspace_root, &mut None)?, - Cow::Borrowed(str), - ), + FilesystemCompletionRoot::String(str) => { + let label = Label::parse(str)?; + ( + self.resolve_folder(&label, current_file, workspace_root)?, + Cow::Borrowed(str), + ) + } }; for entry in fs::read_dir(from_path)? { @@ -665,18 +646,14 @@ impl LspContext for BazelContext { current_file: &LspUrl, workspace_root: Option<&std::path::Path>, ) -> anyhow::Result { - let mut presumed_filename = None; - let folder = - self.resolve_folder(path, current_file, workspace_root, &mut presumed_filename)?; + let label = Label::parse(path)?; + + let folder = self.resolve_folder(&label, current_file, workspace_root)?; // Try the presumed filename first, and check if it exists. - if let Some(presumed_filename) = presumed_filename { - let path = folder.join(presumed_filename); - if path.exists() { - return Ok(Url::from_file_path(path).unwrap().try_into()?); - } - } else { - return Err(ResolveLoadError::CannotParsePath(path.to_owned()).into()); + let presumed_path = folder.join(label.name); + if presumed_path.exists() { + return Ok(Url::from_file_path(presumed_path).unwrap().try_into()?); } // If the presumed filename doesn't exist, try to find a build file from the build system @@ -768,10 +745,12 @@ impl LspContext for BazelContext { location_finder: if same_filename { None } else { - let literal = literal.to_owned(); - Some(Box::new(move |ast| { - Ok(ast.find_function_call_with_name(&literal)) - })) + match Label::parse(literal) { + Err(_) => None, + Ok(label) => Some(Box::new(move |ast| { + Ok(ast.find_function_call_with_name(&label.name)) + })), + } }, }) }) diff --git a/starlark-rust/starlark_bin/bin/bazel/label.rs b/starlark-rust/starlark_bin/bin/bazel/label.rs new file mode 100644 index 0000000000000..1f89026f5e60d --- /dev/null +++ b/starlark-rust/starlark_bin/bin/bazel/label.rs @@ -0,0 +1,280 @@ +/* + * Copyright 2019 The Starlark in Rust Authors. + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Module for parsing bazel labels + +use std::fmt; + +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct Label { + // The repository can be omitted, in which case the label is relative to the current repository + pub repo: Option, + // The package can be omitted, in which case the label is relative to the current package + pub package: Option, + pub name: String, +} + +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct LabelRepo { + pub name: String, + pub is_canonical: bool, +} + +#[derive(thiserror::Error, Debug)] +#[error("Unable to parse the label `{}`", .label)] +pub struct LabelParseError { + label: String, +} + +impl fmt::Display for Label { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(repo) = &self.repo { + fmt::Display::fmt(&repo, f)?; + } + + if let Some(package) = &self.package { + f.write_str("//")?; + f.write_str(&package)?; + } + + f.write_str(":")?; + f.write_str(&self.name)?; + + Ok(()) + } +} + +impl fmt::Display for LabelRepo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(if self.is_canonical { "@@" } else { "@" })?; + f.write_str(&self.name)?; + + Ok(()) + } +} + +impl Label { + pub fn parse(label: &str) -> Result { + match label.split_once("//") { + Some((repo_part, rest)) => { + let repo = if repo_part == "" { + None + } else { + Some(Self::parse_repo(repo_part).ok_or_else(|| LabelParseError { + label: label.to_owned(), + })?) + }; + + let (package, name) = rest.split_once(':').unwrap_or_else(|| { + // Here the name is implicit, and comes from the last component of the package name + if let Some((index, _)) = rest.rmatch_indices('/').last() { + (rest, &rest[index + 1..]) + } else { + (rest, rest) + } + }); + + Ok(Label { + name: name.to_owned(), + package: Some(package.to_owned()), + repo, + }) + } + // Either we have a repo only (@foo or @@foo), or just a name (foo or :foo) + None => { + if let Some(repo) = Self::parse_repo(label) { + Ok(Label { + name: repo.name.to_owned(), + repo: Some(repo), + package: Some("".to_owned()), + }) + } else { + let name = label.strip_prefix(':').unwrap_or(label); + + Ok(Label { + repo: None, + name: name.to_owned(), + package: None, + }) + } + } + } + } + + fn parse_repo(repo: &str) -> Option { + if let Some(repo_name) = repo.strip_prefix("@@") { + Some(LabelRepo { + name: repo_name.to_owned(), + is_canonical: true, + }) + } else if let Some(repo_name) = repo.strip_prefix('@') { + Some(LabelRepo { + name: repo_name.to_owned(), + is_canonical: false, + }) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::Label; + use crate::bazel::label::LabelRepo; + + #[test] + fn test_parsing_repo_only_labels() { + assert_eq!( + Label::parse("@foo").unwrap(), + Label { + repo: Some(LabelRepo { + is_canonical: false, + name: "foo".to_owned(), + }), + package: Some("".to_owned()), + name: "foo".to_owned(), + } + ); + + assert_eq!( + Label::parse("@@foo").unwrap(), + Label { + repo: Some(LabelRepo { + is_canonical: true, + name: "foo".to_owned(), + }), + package: Some("".to_owned()), + name: "foo".to_owned(), + } + ); + } + + #[test] + fn test_parsing_name_only_labels() { + assert_eq!( + Label::parse("foo").unwrap(), + Label { + repo: None, + package: None, + name: "foo".to_owned(), + } + ); + + assert_eq!( + Label::parse(":foo").unwrap(), + Label { + repo: None, + package: None, + name: "foo".to_owned(), + } + ); + } + + #[test] + fn test_full_labels() { + assert_eq!( + Label::parse("//foo/bar:baz").unwrap(), + Label { + repo: None, + package: Some("foo/bar".to_owned()), + name: "baz".to_owned(), + } + ); + + assert_eq!( + Label::parse("@foo//foo/bar:baz").unwrap(), + Label { + repo: Some(LabelRepo { + name: "foo".to_owned(), + is_canonical: false + }), + package: Some("foo/bar".to_owned()), + name: "baz".to_owned(), + } + ); + + assert_eq!( + Label::parse("@@foo//foo/bar:baz").unwrap(), + Label { + repo: Some(LabelRepo { + name: "foo".to_owned(), + is_canonical: true + }), + package: Some("foo/bar".to_owned()), + name: "baz".to_owned(), + } + ); + } + + #[test] + fn test_labels_with_implicit_name() { + assert_eq!( + Label::parse("@foo//bar/baz").unwrap(), + Label { + repo: Some(LabelRepo { + name: "foo".to_owned(), + is_canonical: false + }), + package: Some("bar/baz".to_owned()), + name: "baz".to_owned(), + } + ); + + assert_eq!( + Label::parse("@foo//bar").unwrap(), + Label { + repo: Some(LabelRepo { + name: "foo".to_owned(), + is_canonical: false + }), + package: Some("bar".to_owned()), + name: "bar".to_owned(), + } + ); + + assert_eq!( + Label::parse("@@foo//bar").unwrap(), + Label { + repo: Some(LabelRepo { + name: "foo".to_owned(), + is_canonical: true + }), + package: Some("bar".to_owned()), + name: "bar".to_owned(), + } + ); + } + + #[test] + fn test_invalid_labels() { + assert!(Label::parse("foo//bar/baz").is_err()); + } + + #[test] + fn test_displaying_labels() { + assert_eq!(format!("{}", Label::parse(":foo.bzl").unwrap()), ":foo.bzl"); + assert_eq!( + format!("{}", Label::parse("@foo//bar/baz:qux").unwrap()), + "@foo//bar/baz:qux" + ); + assert_eq!( + format!("{}", Label::parse("//foo/bar").unwrap()), + "//foo/bar:bar" + ); + } +} diff --git a/starlark-rust/starlark_bin/bin/main.rs b/starlark-rust/starlark_bin/bin/main.rs index 4579615365bd9..10b0480fe424c 100644 --- a/starlark-rust/starlark_bin/bin/main.rs +++ b/starlark-rust/starlark_bin/bin/main.rs @@ -17,6 +17,7 @@ // Disagree these are good hints #![allow(clippy::type_complexity)] +#![allow(clippy::manual_map)] use std::ffi::OsStr; use std::fmt; @@ -51,7 +52,7 @@ mod dap; mod eval; #[derive(Debug, Parser)] -#[command(name = "starlark", about = "Evaluate Starlark code")] +#[command(name = "starlark", about = "Evaluate Starlark code", version)] struct Args { #[arg( long = "lsp", diff --git a/starlark-rust/starlark_bin/src/lib.rs b/starlark-rust/starlark_bin/src/lib.rs deleted file mode 100644 index 450ea8a01a754..0000000000000 --- a/starlark-rust/starlark_bin/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2019 The Starlark in Rust Authors. - * Copyright (c) Facebook, Inc. and its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! `starlark` binary. diff --git a/starlark-rust/starlark_derive/Cargo.toml b/starlark-rust/starlark_derive/Cargo.toml index 5ef4f0a42f324..06ea1a52f3b68 100644 --- a/starlark-rust/starlark_derive/Cargo.toml +++ b/starlark-rust/starlark_derive/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" license = "Apache-2.0" name = "starlark_derive" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [lib] proc-macro = true diff --git a/starlark-rust/starlark_derive/src/module/parse/fun.rs b/starlark-rust/starlark_derive/src/module/parse/fun.rs index a3e52e3aa6650..b108d92753688 100644 --- a/starlark-rust/starlark_derive/src/module/parse/fun.rs +++ b/starlark-rust/starlark_derive/src/module/parse/fun.rs @@ -252,10 +252,7 @@ fn is_anyhow_or_starlark_result(t: &Type) -> bool { None => return false, Some(t) => t, }; - match t { - GenericArgument::Type(_) => true, - _ => false, - } + matches!(t, GenericArgument::Type(_)) } // Add a function to the `GlobalsModule` named `globals_builder`. diff --git a/starlark-rust/starlark_lsp/Cargo.toml b/starlark-rust/starlark_lsp/Cargo.toml index b656afb44b251..73f0eb3105cc7 100644 --- a/starlark-rust/starlark_lsp/Cargo.toml +++ b/starlark-rust/starlark_lsp/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["starlark", "skylark", "language", "interpreter"] license = "Apache-2.0" name = "starlark_lsp" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [dependencies] anyhow = "1.0.65" @@ -25,8 +25,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0.36" -starlark = { version = "0.10.0", path = "../starlark" } -starlark_syntax = { version = "0.10.0", path = "../starlark_syntax" } +starlark = { version = "0.12.0", path = "../starlark" } +starlark_syntax = { version = "0.12.0", path = "../starlark_syntax" } [dev-dependencies] regex = "1.5.4" diff --git a/starlark-rust/starlark_lsp/src/bind.rs b/starlark-rust/starlark_lsp/src/bind.rs index a43dfddb9ec66..f25690e25a25d 100644 --- a/starlark-rust/starlark_lsp/src/bind.rs +++ b/starlark-rust/starlark_lsp/src/bind.rs @@ -343,7 +343,7 @@ pub(crate) fn scope(module: &AstModule) -> Scope { } #[cfg(test)] -mod test { +mod tests { use std::iter; use starlark::codemap::Pos; diff --git a/starlark-rust/starlark_lsp/src/definition.rs b/starlark-rust/starlark_lsp/src/definition.rs index ee836cdd452af..1d19c3aba23f6 100644 --- a/starlark-rust/starlark_lsp/src/definition.rs +++ b/starlark-rust/starlark_lsp/src/definition.rs @@ -807,7 +807,7 @@ pub(crate) mod helpers { } #[cfg(test)] -mod test { +mod tests { use textwrap::dedent; use super::helpers::*; diff --git a/starlark-rust/starlark_lsp/src/lib.rs b/starlark-rust/starlark_lsp/src/lib.rs index afe390571d571..496a94f93368d 100644 --- a/starlark-rust/starlark_lsp/src/lib.rs +++ b/starlark-rust/starlark_lsp/src/lib.rs @@ -18,6 +18,9 @@ //! The server that allows IDEs to evaluate and interpret starlark code according //! to the [Language Server Protocol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/). +// Lints that don't necessarily make sense +#[allow(clippy::needless_lifetimes)] +#[allow(clippy::type_complexity)] mod bind; pub mod completion; mod definition; diff --git a/starlark-rust/starlark_lsp/src/server.rs b/starlark-rust/starlark_lsp/src/server.rs index abe57eed6c1c3..e2bd17b135d4e 100644 --- a/starlark-rust/starlark_lsp/src/server.rs +++ b/starlark-rust/starlark_lsp/src/server.rs @@ -410,20 +410,10 @@ impl Backend { trigger_characters: Some(vec![ // e.g. function call "(".to_owned(), - // e.g. list creation, function call - ",".to_owned(), - // e.g. when typing a load path - "/".to_owned(), - // e.g. dict creation - ":".to_owned(), // e.g. variable assignment "=".to_owned(), - // e.g. list creation - "[".to_owned(), // e.g. string literal (load path, target name) "\"".to_owned(), - // don't lose autocomplete when typing a space, e.g. after a comma - " ".to_owned(), ]), ..Default::default() }), @@ -1008,10 +998,7 @@ impl Backend { [ // Actual keywords "and", "else", "load", "break", "for", "not", "continue", "if", "or", "def", "in", - "pass", "elif", "return", "lambda", // - // Reserved words - "as", "import", "is", "class", "nonlocal", "del", "raise", "except", "try", "finally", - "while", "from", "with", "global", "yield", + "pass", "elif", "return", "lambda", ] .into_iter() .map(|keyword| CompletionItem { @@ -1387,7 +1374,7 @@ where // TODO(nmj): Some of the windows tests get a bit flaky, especially around // some paths. Revisit later. #[cfg(all(test, not(windows)))] -mod test { +mod tests { use std::path::Path; use std::path::PathBuf; diff --git a/starlark-rust/starlark_map/BUCK b/starlark-rust/starlark_map/BUCK index 05c2a063acbc4..cea631ede56a1 100644 --- a/starlark-rust/starlark_map/BUCK +++ b/starlark-rust/starlark_map/BUCK @@ -14,7 +14,7 @@ rust_library( ], deps = [ "fbsource//third-party/rust:equivalent", - "fbsource//third-party/rust:fnv", + "fbsource//third-party/rust:fxhash", "fbsource//third-party/rust:hashbrown", "fbsource//third-party/rust:serde", "//buck2/allocative/allocative:allocative", diff --git a/starlark-rust/starlark_map/Cargo.toml b/starlark-rust/starlark_map/Cargo.toml index 0faab26428958..312ccdedae223 100644 --- a/starlark-rust/starlark_map/Cargo.toml +++ b/starlark-rust/starlark_map/Cargo.toml @@ -8,15 +8,15 @@ edition = "2021" license = "Apache-2.0" name = "starlark_map" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [dependencies] allocative = { workspace = true, features = ["hashbrown"] } dupe = { workspace = true } equivalent = { workspace = true } -fnv = "1.0.7" -hashbrown = { version = "0.12.3", features = ["raw"] } +fxhash = "0.2.1" +hashbrown = { version = "0.14.3", features = ["raw"] } serde = { version = "1.0", features = ["derive"] } [dev-dependencies] diff --git a/starlark-rust/starlark_map/src/hasher.rs b/starlark-rust/starlark_map/src/hasher.rs index a85422637d741..0443c6599ad78 100644 --- a/starlark-rust/starlark_map/src/hasher.rs +++ b/starlark-rust/starlark_map/src/hasher.rs @@ -19,7 +19,7 @@ use std::hash::BuildHasher; use std::hash::Hasher; use dupe::Dupe; -use fnv::FnvHasher; +use fxhash::FxHasher64; use crate::hash_value::StarlarkHashValue; @@ -27,7 +27,10 @@ use crate::hash_value::StarlarkHashValue; /// /// Starlark relies on stable hashing, and this is the hasher. #[derive(Default)] -pub struct StarlarkHasher(FnvHasher); +pub struct StarlarkHasher( + // TODO(nga): `FxHasher64` is endian-dependent, this is not right. + FxHasher64, +); impl StarlarkHasher { /// Creates a new hasher. diff --git a/starlark-rust/starlark_map/src/ordered_map.rs b/starlark-rust/starlark_map/src/ordered_map.rs index 0237183ebfd94..1f05166c0be8c 100644 --- a/starlark-rust/starlark_map/src/ordered_map.rs +++ b/starlark-rust/starlark_map/src/ordered_map.rs @@ -89,7 +89,7 @@ impl OrderedMap { /// Get a reference to the value associated with the given key. #[inline] - pub fn get(&self, k: &Q) -> Option<&V> + pub fn get<'a, Q>(&'a self, k: &Q) -> Option<&'a V> where Q: Hash + Equivalent + ?Sized, { @@ -98,7 +98,7 @@ impl OrderedMap { /// Get a mutable reference to the value associated with the given key. #[inline] - pub fn get_mut(&mut self, k: &Q) -> Option<&mut V> + pub fn get_mut<'a, Q>(&'a mut self, k: &Q) -> Option<&'a mut V> where Q: Hash + Equivalent + ?Sized, { diff --git a/starlark-rust/starlark_map/src/ordered_set.rs b/starlark-rust/starlark_map/src/ordered_set.rs index db778e84bd7f2..7c6a21d251f8d 100644 --- a/starlark-rust/starlark_map/src/ordered_set.rs +++ b/starlark-rust/starlark_map/src/ordered_set.rs @@ -113,7 +113,7 @@ impl OrderedSet { /// Iterate over the elements. #[inline] - pub fn iter(&self) -> small_set::Iter { + pub fn iter(&self) -> Iter { self.0.iter() } @@ -138,6 +138,15 @@ impl OrderedSet { self.0.insert(value) } + /// Insert an element into the set assuming it is not already present. + #[inline] + pub fn insert_unique_unchecked(&mut self, value: T) + where + T: Hash, + { + self.0.insert_unique_unchecked(value) + } + /// Insert an element if element is not present in the set, /// otherwise return the element. #[inline] @@ -191,6 +200,11 @@ impl OrderedSet { { self.0.union(&other.0) } + + /// Reverse the iteration order of the set. + pub fn reverse(&mut self) { + self.0.reverse(); + } } impl Default for OrderedSet { @@ -230,9 +244,14 @@ impl Hash for OrderedSet { } } +/// Iterator returned by `iter`. +pub type Iter<'a, T> = small_set::Iter<'a, T>; +/// Iterator returned by `into_iter`. +pub type IntoIter = small_set::IntoIter; + impl IntoIterator for OrderedSet { type Item = T; - type IntoIter = small_set::IntoIter; + type IntoIter = IntoIter; #[inline] fn into_iter(self) -> Self::IntoIter { @@ -242,7 +261,7 @@ impl IntoIterator for OrderedSet { impl<'a, T> IntoIterator for &'a OrderedSet { type Item = &'a T; - type IntoIter = small_set::Iter<'a, T>; + type IntoIter = Iter<'a, T>; #[inline] fn into_iter(self) -> Self::IntoIter { @@ -267,6 +286,16 @@ where } } +impl Extend for OrderedSet +where + T: Eq + Hash, +{ + #[inline] + fn extend>(&mut self, iter: I) { + self.0.extend(iter) + } +} + #[cfg(test)] mod tests { use std::cell::Cell; diff --git a/starlark-rust/starlark_map/src/small_map/mod.rs b/starlark-rust/starlark_map/src/small_map/mod.rs index a75ceb381cf70..52c10033a4dd5 100644 --- a/starlark-rust/starlark_map/src/small_map/mod.rs +++ b/starlark-rust/starlark_map/src/small_map/mod.rs @@ -710,6 +710,20 @@ impl SmallMap { { self.entries.hash_ordered(state) } + + /// Reverse the iteration order of the map. + pub fn reverse(&mut self) { + self.entries.reverse(); + if let Some(index) = &mut self.index { + let len = self.entries.len(); + // Safety: iterator does not escape. + unsafe { + for entry in index.iter() { + *entry.as_mut() = len - 1 - *entry.as_ref(); + } + } + } + } } /// Reference to the actual entry in the map. @@ -820,6 +834,7 @@ where where V: Default, { + #[allow(clippy::unwrap_or_default)] // defining or_default self.or_insert_with(V::default) } @@ -919,7 +934,7 @@ where /// ``` /// use starlark_map::smallmap; /// -/// let map = smallmap!{ +/// let map = smallmap! { /// "a" => 1, /// "b" => 2, /// }; @@ -1016,6 +1031,7 @@ mod tests { } #[test] + #[allow(clippy::map_identity)] fn few_entries() { let entries1 = [(0, 'a'), (1, 'b')]; let m1 = entries1.iter().copied().collect::>(); @@ -1325,4 +1341,49 @@ mod tests { mp ); } + + #[test] + fn test_reverse_small() { + let mut map = SmallMap::new(); + map.insert("a".to_owned(), "b".to_owned()); + map.insert("c".to_owned(), "d".to_owned()); + map.reverse(); + + assert_eq!(Some("b"), map.get("a").map(|s| s.as_str())); + assert_eq!(Some("d"), map.get("c").map(|s| s.as_str())); + assert_eq!( + vec![ + ("c".to_owned(), "d".to_owned()), + ("a".to_owned(), "b".to_owned()) + ], + map.into_iter().collect::>() + ); + } + + #[test] + fn test_reverse_large() { + let mut map = SmallMap::new(); + for i in 0..100 { + map.insert(i.to_string(), (i * 10).to_string()); + } + + let expected = map + .iter() + .rev() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + map.reverse(); + + for i in 0..100 { + assert_eq!(Some(&(i * 10).to_string()), map.get(&i.to_string())); + } + + assert_eq!( + expected, + map.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>() + ); + } } diff --git a/starlark-rust/starlark_map/src/small_set/mod.rs b/starlark-rust/starlark_map/src/small_set/mod.rs index 6998fc95944fc..332ea6cf4a5db 100644 --- a/starlark-rust/starlark_map/src/small_set/mod.rs +++ b/starlark-rust/starlark_map/src/small_set/mod.rs @@ -147,7 +147,7 @@ impl SmallSet { #[inline] pub fn insert_unique_unchecked(&mut self, key: T) where - T: Hash + Eq, + T: Hash, { self.0.insert_unique_unchecked(key, ()); } @@ -400,6 +400,11 @@ impl SmallSet { { self.0.hash_ordered(state) } + + /// Reverse the iteration order of the set. + pub fn reverse(&mut self) { + self.0.reverse(); + } } impl<'a, T> IntoIterator for &'a SmallSet { @@ -492,7 +497,7 @@ where /// ``` /// use starlark_map::smallset; /// -/// let set = smallset!{"a", "b"}; +/// let set = smallset! {"a", "b"}; /// assert_eq!(set.contains("a"), true); /// assert_eq!(set.len(), 2); /// assert_eq!(set.contains("c"), false); diff --git a/starlark-rust/starlark_map/src/unordered_map.rs b/starlark-rust/starlark_map/src/unordered_map.rs index a2fc2928f671b..0cd3aa91d39c4 100644 --- a/starlark-rust/starlark_map/src/unordered_map.rs +++ b/starlark-rust/starlark_map/src/unordered_map.rs @@ -28,15 +28,10 @@ use hashbrown::raw::Bucket; use hashbrown::raw::RawTable; use crate::Equivalent; +use crate::Hashed; +use crate::StarlarkHashValue; use crate::StarlarkHasher; -#[inline] -fn compute_hash(k: &Q) -> u64 { - let mut hasher = StarlarkHasher::new(); - k.hash(&mut hasher); - hasher.finish() -} - /// Hash map which does not expose any insertion order-specific behavior /// (except `Debug`). #[derive(Clone, Allocative)] @@ -80,9 +75,19 @@ impl UnorderedMap { where Q: Hash + Equivalent + ?Sized, { - let hash = compute_hash(k); + let k = Hashed::new(k); + self.get_hashed(k) + } + + /// Get a reference to the value associated with the given key. + #[inline] + pub fn get_hashed(&self, key: Hashed<&Q>) -> Option<&V> + where + Q: Equivalent + ?Sized, + { + let hash = key.hash().promote(); self.0 - .get(hash, |(next_k, _v)| k.equivalent(next_k)) + .get(hash, |(next_k, _v)| key.key().equivalent(next_k)) .map(|(_, v)| v) } @@ -92,7 +97,7 @@ impl UnorderedMap { where Q: Hash + Equivalent + ?Sized, { - let hash = compute_hash(k); + let hash = StarlarkHashValue::new(k).promote(); self.0 .get_mut(hash, |(next_k, _v)| k.equivalent(next_k)) .map(|(_, v)| v) @@ -107,20 +112,31 @@ impl UnorderedMap { self.get(k).is_some() } + /// Does the map contain the specified key? + #[inline] + pub fn contains_key_hashed(&self, key: Hashed<&Q>) -> bool + where + Q: Equivalent + ?Sized, + { + self.get_hashed(key).is_some() + } + /// Insert an entry into the map. #[inline] pub fn insert(&mut self, k: K, v: V) -> Option where K: Hash + Eq, { - let hash = compute_hash(&k); - if let Some((_k, existing_value)) = - self.0.get_mut(hash, |(next_k, _v)| k.equivalent(next_k)) - { - Some(mem::replace(existing_value, v)) - } else { - self.0.insert(hash, (k, v), |(k, _v)| compute_hash(k)); - None + let k = Hashed::new(k); + match self.raw_entry_mut().from_key_hashed(k.as_ref()) { + RawEntryMut::Occupied(mut e) => { + let old = e.insert(v); + Some(old) + } + RawEntryMut::Vacant(e) => { + e.insert_hashed(k, v); + None + } } } @@ -130,10 +146,30 @@ impl UnorderedMap { where Q: Hash + Equivalent + ?Sized, { - let hash = compute_hash(k); - self.0 - .remove_entry(hash, |(next_k, _v)| k.equivalent(next_k)) - .map(|(_, v)| v) + match self.raw_entry_mut().from_key(k) { + RawEntryMut::Occupied(e) => Some(e.remove()), + RawEntryMut::Vacant(_) => None, + } + } + + /// Preserve only the elements specified by the predicate. + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, + { + // TODO(nga): update hashbrown and use safe `HashTable` instead of this heavily unsafe code: + // https://docs.rs/hashbrown/latest/hashbrown/struct.HashTable.html + + // Unsafe code is copy-paste from `hashbrown` crate: + // https://github.com/rust-lang/hashbrown/blob/f2e62124cd947b5e2309dd6a24c7e422932aae97/src/map.rs#L923 + unsafe { + for item in self.0.iter() { + let (k, v) = item.as_mut(); + if !f(k, v) { + self.0.erase(item); + } + } + } } /// Get an entry in the map for in-place manipulation. @@ -142,7 +178,7 @@ impl UnorderedMap { where K: Hash + Eq, { - let hash = compute_hash(&k); + let hash = StarlarkHashValue::new(&k).promote(); if let Some(bucket) = self.0.find(hash, |(next_k, _v)| k.equivalent(next_k)) { Entry::Occupied(OccupiedEntry { _map: self, bucket }) } else { @@ -154,20 +190,51 @@ impl UnorderedMap { } } + /// Lower-level access to the entry API. + #[inline] + pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut { + RawEntryBuilderMut { map: self } + } + /// Clear the map, removing all entries. #[inline] pub fn clear(&mut self) { self.0.clear(); } - /// This function is private. + /// Entries in the map, in arbitrary order. #[inline] - pub(crate) fn iter(&self) -> impl ExactSizeIterator { + pub fn entries_unordered(&self) -> impl ExactSizeIterator { unsafe { self.0.iter().map(|e| (&e.as_ref().0, &e.as_ref().1)) } } - /// This function is private. - pub(crate) fn into_iter(self) -> impl ExactSizeIterator { + /// Entries in the map, in arbitrary order. + #[inline] + pub fn entries_unordered_mut(&mut self) -> impl ExactSizeIterator { + unsafe { self.0.iter().map(|e| (&e.as_ref().0, &mut e.as_mut().1)) } + } + + /// Keys in the map, in arbitrary order. + #[inline] + pub fn keys_unordered(&self) -> impl ExactSizeIterator { + self.entries_unordered().map(|(k, _v)| k) + } + + /// Values in the map, in arbitrary order. + #[inline] + pub fn values_unordered(&self) -> impl ExactSizeIterator { + self.entries_unordered().map(|(_k, v)| v) + } + + /// Values in the map, in arbitrary order. + #[inline] + pub fn values_unordered_mut(&mut self) -> impl ExactSizeIterator { + self.entries_unordered_mut().map(|(_k, v)| v) + } + + /// Into entries, in arbitrary order. + #[inline] + pub(crate) fn into_entries_unordered(self) -> impl ExactSizeIterator { self.0.into_iter() } @@ -176,7 +243,7 @@ impl UnorderedMap { where K: Ord, { - let mut entries = Vec::from_iter(self.iter()); + let mut entries = Vec::from_iter(self.entries_unordered()); entries.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); entries } @@ -186,7 +253,7 @@ impl UnorderedMap { where K: Hash + Eq, { - self.into_iter().collect() + self.into_entries_unordered().collect() } /// Apply the function to value. @@ -195,7 +262,7 @@ impl UnorderedMap { K: Hash + Eq, { let mut map = UnorderedMap::with_capacity(self.len()); - for (k, v) in self.into_iter() { + for (k, v) in self.into_entries_unordered() { map.insert(k, f(v)); } map @@ -213,14 +280,17 @@ impl + Hash> Index<&Q> for UnorderedMap { impl Debug for UnorderedMap { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_map().entries(self.iter()).finish() + f.debug_map().entries(self.entries_unordered()).finish() } } impl PartialEq for UnorderedMap { #[inline] fn eq(&self, other: &Self) -> bool { - self.len() == other.len() && self.iter().all(|(k, v)| other.get(k) == Some(v)) + self.len() == other.len() + && self + .entries_unordered() + .all(|(k, v)| other.get(k) == Some(v)) } } @@ -230,7 +300,7 @@ impl Hash for UnorderedMap { fn hash(&self, state: &mut H) { self.len().hash(state); let mut sum: u64 = 0; - for (k, v) in self.iter() { + for (k, v) in self.entries_unordered() { let mut hasher = StarlarkHasher::new(); (k, v).hash(&mut hasher); sum = sum.wrapping_add(hasher.finish()); @@ -273,28 +343,167 @@ pub enum Entry<'a, K, V> { impl<'a, K: Eq + Hash, V> VacantEntry<'a, K, V> { /// Insert a value into the map. + #[inline] pub fn insert(self, value: V) { - self.map - .0 - .insert(self.hash, (self.key, value), |(k, _v)| compute_hash(k)); + self.map.0.insert(self.hash, (self.key, value), |(k, _v)| { + StarlarkHashValue::new(k).promote() + }); } } impl<'a, K, V> OccupiedEntry<'a, K, V> { /// Remove the entry from the map. + #[inline] pub fn get(&self) -> &V { unsafe { &self.bucket.as_ref().1 } } /// Get a reference to the value associated with the entry. + #[inline] pub fn get_mut(&mut self) -> &mut V { unsafe { &mut self.bucket.as_mut().1 } } /// Replace the value associated with the entry. + #[inline] + pub fn insert(&mut self, value: V) -> V { + mem::replace(self.get_mut(), value) + } +} + +/// Builder for [`RawEntryMut`]. +pub struct RawEntryBuilderMut<'a, K, V> { + map: &'a mut UnorderedMap, +} + +impl<'a, K, V> RawEntryBuilderMut<'a, K, V> { + /// Find an entry by key. + #[inline] + pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V> + where + Q: Hash + Equivalent + ?Sized, + { + let k = Hashed::new(k); + self.from_key_hashed(k) + } + + /// Find an entry by hashed key. + #[inline] + pub fn from_key_hashed(self, k: Hashed<&Q>) -> RawEntryMut<'a, K, V> + where + Q: Equivalent + ?Sized, + { + self.from_hash(k.hash(), |next_k| k.key().equivalent(next_k)) + } + + /// Find an entry by hash and equality function. + #[inline] + pub fn from_hash(self, hash: StarlarkHashValue, mut is_match: F) -> RawEntryMut<'a, K, V> + where + F: for<'b> FnMut(&'b K) -> bool, + { + let hash = hash.promote(); + if let Some(bucket) = self.map.0.find(hash, |(next_k, _v)| is_match(next_k)) { + RawEntryMut::Occupied(RawOccupiedEntryMut { + map: self.map, + bucket, + }) + } else { + RawEntryMut::Vacant(RawVacantEntryMut { map: self.map }) + } + } +} + +/// Occupied entry. +pub struct RawOccupiedEntryMut<'a, K, V> { + map: &'a mut UnorderedMap, + bucket: Bucket<(K, V)>, +} + +/// Vacant entry. +pub struct RawVacantEntryMut<'a, K, V> { + map: &'a mut UnorderedMap, +} + +/// Raw entry. +pub enum RawEntryMut<'a, K, V> { + /// Occupied entry. + Occupied(RawOccupiedEntryMut<'a, K, V>), + /// Vacant entry. + Vacant(RawVacantEntryMut<'a, K, V>), +} + +impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { + /// Replace the value associated with the entry. + #[inline] pub fn insert(&mut self, value: V) -> V { mem::replace(self.get_mut(), value) } + + /// Replace the key associated with the entry. + #[inline] + pub fn insert_key(&mut self, key: K) -> K { + mem::replace(self.key_mut(), key) + } + + /// Get a reference to the value associated with the entry. + #[inline] + pub fn get(&self) -> &V { + unsafe { &self.bucket.as_ref().1 } + } + + /// Get a reference to the value associated with the entry. + #[inline] + pub fn get_mut(&mut self) -> &mut V { + unsafe { &mut self.bucket.as_mut().1 } + } + + /// Get a reference to the key associated with the entry. + #[inline] + pub fn key_mut(&mut self) -> &mut K { + unsafe { &mut self.bucket.as_mut().0 } + } + + /// Remove the entry, return the value. + #[inline] + pub fn remove(self) -> V { + self.remove_entry().1 + } + + /// Remove the entry, return the key and value. + #[inline] + pub fn remove_entry(self) -> (K, V) { + unsafe { self.map.0.remove(self.bucket).0 } + } +} + +impl<'a, K, V> RawVacantEntryMut<'a, K, V> { + /// Insert entry. + /// + /// Not this function computes the hash of the key. + #[inline] + pub fn insert(self, key: K, value: V) -> (&'a mut K, &'a mut V) + where + K: Hash, + { + let key = Hashed::new(key); + self.insert_hashed(key, value) + } + + /// Insert entry. + #[inline] + pub fn insert_hashed(self, key: Hashed, value: V) -> (&'a mut K, &'a mut V) + where + K: Hash, + { + let (k, v) = + self.map + .0 + .insert_entry(key.hash().promote(), (key.into_key(), value), |(k, _v)| { + StarlarkHashValue::new(k).promote() + }); + (k, v) + } } #[cfg(test)] @@ -349,4 +558,26 @@ mod tests { map.insert(3, 4); assert_eq!(map.entries_sorted(), vec![(&1, &2), (&3, &4), (&5, &6)]); } + + #[test] + fn test_retain() { + let mut map = UnorderedMap::new(); + for i in 0..1000 { + map.insert(format!("key{}", i), format!("value{}", i)); + } + + map.retain(|k, v| { + v.push('x'); + k.ends_with('0') + }); + + assert_eq!(100, map.len()); + for i in 0..1000 { + if i % 10 == 0 { + assert_eq!(format!("value{}x", i), map[&format!("key{}", i)]); + } else { + assert!(!map.contains_key(&format!("key{}", i))); + } + } + } } diff --git a/starlark-rust/starlark_map/src/unordered_set.rs b/starlark-rust/starlark_map/src/unordered_set.rs index 1945d56049fe0..631b62320f3b3 100644 --- a/starlark-rust/starlark_map/src/unordered_set.rs +++ b/starlark-rust/starlark_map/src/unordered_set.rs @@ -21,15 +21,25 @@ use std::hash::Hash; use allocative::Allocative; +use crate::unordered_map; use crate::unordered_map::UnorderedMap; use crate::Equivalent; +use crate::Hashed; +use crate::StarlarkHashValue; /// `HashSet` that does not expose insertion order. -#[derive(Clone, Allocative, Debug, Default)] +#[derive(Clone, Allocative, Debug)] pub struct UnorderedSet { map: UnorderedMap, } +impl Default for UnorderedSet { + #[inline] + fn default() -> UnorderedSet { + UnorderedSet::new() + } +} + impl UnorderedSet { /// Create a new empty set. #[inline] @@ -83,9 +93,26 @@ impl UnorderedSet { self.map.contains_key(value) } + /// Does the set contain the specified value? + #[inline] + pub fn contains_hashed(&self, value: Hashed<&Q>) -> bool + where + Q: Equivalent + ?Sized, + { + self.map.contains_key_hashed(value) + } + + /// Lower-level access to the underlying map. + #[inline] + pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut { + RawEntryBuilderMut { + entry: self.map.raw_entry_mut(), + } + } + /// This function is private. fn iter(&self) -> impl Iterator { - self.map.iter().map(|(k, _)| k) + self.map.entries_unordered().map(|(k, _)| k) } /// Get the entries in the set, sorted. @@ -117,6 +144,101 @@ impl FromIterator for UnorderedSet { } } +/// Builder for [`RawEntryMut`]. +pub struct RawEntryBuilderMut<'a, T> { + entry: unordered_map::RawEntryBuilderMut<'a, T, ()>, +} + +impl<'a, T> RawEntryBuilderMut<'a, T> { + /// Find the entry for a key. + #[inline] + pub fn from_entry(self, entry: &Q) -> RawEntryMut<'a, T> + where + Q: Hash + Equivalent + ?Sized, + { + let entry = Hashed::new(entry); + self.from_entry_hashed(entry) + } + + /// Find the entry for a key. + #[inline] + pub fn from_entry_hashed(self, entry: Hashed<&Q>) -> RawEntryMut<'a, T> + where + Q: ?Sized + Equivalent, + { + self.from_hash(entry.hash(), |k| entry.key().equivalent(k)) + } + + /// Find the entry by hash and equality function. + #[inline] + pub fn from_hash(self, hash: StarlarkHashValue, is_match: F) -> RawEntryMut<'a, T> + where + F: for<'b> FnMut(&'b T) -> bool, + { + match self.entry.from_hash(hash, is_match) { + unordered_map::RawEntryMut::Occupied(e) => { + RawEntryMut::Occupied(RawOccupiedEntryMut { entry: e }) + } + unordered_map::RawEntryMut::Vacant(e) => { + RawEntryMut::Vacant(RawVacantEntryMut { entry: e }) + } + } + } +} + +/// Reference to an occupied entry in a [`UnorderedSet`]. +pub struct RawOccupiedEntryMut<'a, T> { + entry: unordered_map::RawOccupiedEntryMut<'a, T, ()>, +} + +/// Reference to a vacant entry in a [`UnorderedSet`]. +pub struct RawVacantEntryMut<'a, T> { + entry: unordered_map::RawVacantEntryMut<'a, T, ()>, +} + +/// Reference to an entry in a [`UnorderedSet`]. +pub enum RawEntryMut<'a, T> { + /// Occupied entry. + Occupied(RawOccupiedEntryMut<'a, T>), + /// Vacant entry. + Vacant(RawVacantEntryMut<'a, T>), +} + +impl<'a, T> RawOccupiedEntryMut<'a, T> { + /// Remove the entry. + #[inline] + pub fn remove(self) -> T { + self.entry.remove_entry().0 + } + + /// Replace the entry. + #[inline] + pub fn insert(&mut self, value: T) -> T { + self.entry.insert_key(value) + } +} + +impl<'a, T> RawVacantEntryMut<'a, T> { + /// Insert an entry to the set. This function computes the hash of the key. + #[inline] + pub fn insert(self, value: T) + where + T: Hash, + { + let value = Hashed::new(value); + self.insert_hashed(value); + } + + /// Insert an entry to the set. + #[inline] + pub fn insert_hashed(self, value: Hashed) + where + T: Hash, + { + self.entry.insert_hashed(value, ()); + } +} + #[cfg(test)] mod tests { use crate::unordered_set::UnorderedSet; diff --git a/starlark-rust/starlark_map/src/vec2/mod.rs b/starlark-rust/starlark_map/src/vec2/mod.rs index 645fe23726413..969782b7e6eef 100644 --- a/starlark-rust/starlark_map/src/vec2/mod.rs +++ b/starlark-rust/starlark_map/src/vec2/mod.rs @@ -204,7 +204,7 @@ impl Vec2 { } #[inline] - fn bbb_mut(&mut self) -> &mut [B] { + pub(crate) fn bbb_mut(&mut self) -> &mut [B] { unsafe { slice::from_raw_parts_mut(self.bbb_ptr().as_ptr(), self.len) } } diff --git a/starlark-rust/starlark_map/src/vec_map/mod.rs b/starlark-rust/starlark_map/src/vec_map/mod.rs index b7bbb145c723c..9c4dc65f9f86a 100644 --- a/starlark-rust/starlark_map/src/vec_map/mod.rs +++ b/starlark-rust/starlark_map/src/vec_map/mod.rs @@ -287,4 +287,9 @@ impl VecMap { e.hash(state); } } + + pub(crate) fn reverse(&mut self) { + self.buckets.aaa_mut().reverse(); + self.buckets.bbb_mut().reverse(); + } } diff --git a/starlark-rust/starlark_syntax/Cargo.toml b/starlark-rust/starlark_syntax/Cargo.toml index d4e618fa488f0..1bd7824554cd4 100644 --- a/starlark-rust/starlark_syntax/Cargo.toml +++ b/starlark-rust/starlark_syntax/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["starlark", "skylark", "bazel", "language", "interpreter"] license = "Apache-2.0" name = "starlark_syntax" repository = "https://github.com/facebookexperimental/starlark-rust" -version = "0.10.0" +version = "0.12.0" [build-dependencies] lalrpop = "0.19.7" @@ -32,7 +32,7 @@ thiserror = "1.0.36" allocative = { workspace = true } dupe = { workspace = true } -starlark_map = { version = "0.10.0", path = "../starlark_map" } +starlark_map = { version = "0.12.0", path = "../starlark_map" } [dev-dependencies] serde_json = "1.0" diff --git a/starlark-rust/starlark_syntax/src/error.rs b/starlark-rust/starlark_syntax/src/error.rs index 80e80e0872eba..a514520231eab 100644 --- a/starlark-rust/starlark_syntax/src/error.rs +++ b/starlark-rust/starlark_syntax/src/error.rs @@ -156,6 +156,8 @@ impl fmt::Debug for Error { pub enum ErrorKind { /// An explicit `fail` invocation Fail(anyhow::Error), + /// Starlark call stack overflow. + StackOverflow(anyhow::Error), /// An error approximately associated with a value. /// /// Includes unsupported operations, missing attributes, things of that sort. @@ -164,6 +166,8 @@ pub enum ErrorKind { Function(anyhow::Error), /// Out of scope variables and similar Scope(anyhow::Error), + /// Error when lexing a file + Lexer(anyhow::Error), /// Indicates a logic bug in starlark Internal(anyhow::Error), /// Fallback option @@ -180,9 +184,11 @@ impl ErrorKind { pub fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { Self::Fail(_) => None, + Self::StackOverflow(_) => None, Self::Value(_) => None, Self::Function(_) => None, Self::Scope(_) => None, + Self::Lexer(_) => None, Self::Internal(_) => None, Self::Other(e) => e.source(), } @@ -194,8 +200,10 @@ impl fmt::Debug for ErrorKind { match self { Self::Fail(s) => write!(f, "fail:{}", s), Self::Value(e) => fmt::Debug::fmt(e, f), + Self::StackOverflow(e) => fmt::Debug::fmt(e, f), Self::Function(e) => fmt::Debug::fmt(e, f), Self::Scope(e) => fmt::Debug::fmt(e, f), + Self::Lexer(e) => fmt::Debug::fmt(e, f), Self::Internal(e) => write!(f, "Internal error: {}", e), Self::Other(e) => fmt::Debug::fmt(e, f), } @@ -206,9 +214,11 @@ impl fmt::Display for ErrorKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Fail(s) => write!(f, "fail:{}", s), + Self::StackOverflow(e) => fmt::Display::fmt(e, f), Self::Value(e) => fmt::Display::fmt(e, f), Self::Function(e) => fmt::Display::fmt(e, f), Self::Scope(e) => fmt::Display::fmt(e, f), + Self::Lexer(e) => fmt::Display::fmt(e, f), Self::Internal(e) => write!(f, "Internal error: {}", e), Self::Other(e) => fmt::Display::fmt(e, f), } diff --git a/starlark-rust/starlark_syntax/src/lexer.rs b/starlark-rust/starlark_syntax/src/lexer.rs index 1b7905080771a..d3368f6f3687b 100644 --- a/starlark-rust/starlark_syntax/src/lexer.rs +++ b/starlark-rust/starlark_syntax/src/lexer.rs @@ -59,6 +59,12 @@ pub enum LexemeError { CannotParse(String, u32), } +impl From for crate::error::Error { + fn from(e: LexemeError) -> Self { + crate::error::Error::new(crate::error::ErrorKind::Lexer(anyhow::Error::new(e))) + } +} + type LexemeT = Result<(usize, T, usize), EvalException>; type Lexeme = LexemeT; @@ -101,7 +107,7 @@ impl<'a> Lexer<'a> { } fn err_span(&self, msg: LexemeError, start: usize, end: usize) -> Result { - Err(EvalException::new_anyhow( + Err(EvalException::new( msg.into(), Span::new(Pos::new(start as u32), Pos::new(end as u32)), &self.codemap, @@ -591,7 +597,7 @@ pub enum TokenInt { } impl TokenInt { - pub fn from_str_radix(s: &str, base: u32) -> anyhow::Result { + pub fn from_str_radix(s: &str, base: u32) -> crate::Result { if let Ok(i) = i32::from_str_radix(s, base) { Ok(TokenInt::I32(i)) } else { @@ -960,7 +966,7 @@ pub fn lex_exactly_one_identifier(s: &str) -> Option { } #[cfg(test)] -mod test { +mod tests { use crate::lexer::lex_exactly_one_identifier; #[test] diff --git a/starlark-rust/starlark_syntax/src/syntax/def.rs b/starlark-rust/starlark_syntax/src/syntax/def.rs index 5b6086b33977f..5ec06e862bc86 100644 --- a/starlark-rust/starlark_syntax/src/syntax/def.rs +++ b/starlark-rust/starlark_syntax/src/syntax/def.rs @@ -199,10 +199,9 @@ impl<'a, P: AstPayload> DefParams<'a, P> { if matches!( param.node, ParameterP::Args(..) | ParameterP::KwArgs(..) | ParameterP::NoArgs - ) { - if num_positional.is_none() { - num_positional = Some(i); - } + ) && num_positional.is_none() + { + num_positional = Some(i); } } Ok(DefParams { diff --git a/starlark-rust/starlark_syntax/src/syntax/mod.rs b/starlark-rust/starlark_syntax/src/syntax/mod.rs index 6e70b70f9eb88..7d89d2dfa0c33 100644 --- a/starlark-rust/starlark_syntax/src/syntax/mod.rs +++ b/starlark-rust/starlark_syntax/src/syntax/mod.rs @@ -45,6 +45,9 @@ pub mod validate; #[allow(clippy::trivially_copy_pass_by_ref)] #[allow(clippy::too_many_arguments)] #[allow(clippy::cloned_instead_of_copied)] +#[allow(clippy::type_complexity)] +#[allow(clippy::needless_lifetimes)] +#[allow(clippy::single_match)] #[allow(unused_extern_crates)] #[allow(unused_braces)] diff --git a/starlark-rust/starlark_syntax/src/syntax/module.rs b/starlark-rust/starlark_syntax/src/syntax/module.rs index c749e37fb3f4e..5b996016b4e14 100644 --- a/starlark-rust/starlark_syntax/src/syntax/module.rs +++ b/starlark-rust/starlark_syntax/src/syntax/module.rs @@ -179,10 +179,12 @@ impl AstModule { /// The returned error may contain diagnostic information. For example: /// /// ``` - /// use starlark_syntax::syntax::{AstModule, Dialect}; /// use starlark_syntax::codemap::FileSpan; + /// use starlark_syntax::syntax::AstModule; + /// use starlark_syntax::syntax::Dialect; /// - /// let err: starlark_syntax::Error = AstModule::parse("filename", "\n(unmatched".to_owned(), &Dialect::Standard).unwrap_err(); + /// let err: starlark_syntax::Error = + /// AstModule::parse("filename", "\n(unmatched".to_owned(), &Dialect::Standard).unwrap_err(); /// let span: &FileSpan = err.span().unwrap(); /// assert_eq!(span.to_string(), "filename:2:11"); /// ``` @@ -197,10 +199,7 @@ impl AstModule { dialect, errors: &mut errors, }, - lexer.filter(|t| match t { - Ok((_, Token::Comment(_), _)) => false, - _ => true, - }), + lexer.filter(|t| !matches!(t, Ok((_, Token::Comment(_), _)))), ) { Ok(v) => { if let Some(err) = errors.into_iter().next() { diff --git a/starlark-rust/starlark_syntax/testcases/parse/README.md b/starlark-rust/starlark_syntax/testcases/parse/README.md index 32c0b39254d2a..7f59d8c1bcd0a 100644 --- a/starlark-rust/starlark_syntax/testcases/parse/README.md +++ b/starlark-rust/starlark_syntax/testcases/parse/README.md @@ -1,6 +1,7 @@ # Parsing test cases -A set of `.bzl` files taken from various open-source projects to test the Starlark parser against real world cases. +A set of `.bzl` files taken from various open-source projects to test the +Starlark parser against real world cases. -Files are marked generated, that was done to mute linters. There's -no automation to regenerate these files. +Files are marked generated, that was done to mute linters. There's no automation +to regenerate these files. diff --git a/starlark-rust/vscode/README.md b/starlark-rust/vscode/README.md index c665bbf342aa3..06714851fe935 100644 --- a/starlark-rust/vscode/README.md +++ b/starlark-rust/vscode/README.md @@ -1,18 +1,27 @@ # Starlark VS Code LSP extension -A VSCode LSP extension that talks over stdin/stdout to a binary. This can either be the starlark binary itself, or any binary that has implemented `starlark::lsp::server::LspContext` and runs `starlark::lsp::server::stdio_server()`. +A VSCode LSP extension that talks over stdin/stdout to a binary. This can either +be the starlark binary itself, or any binary that has implemented +`starlark::lsp::server::LspContext` and runs +`starlark::lsp::server::stdio_server()`. -If using another binary, the settings to be aware of are `starlark.lspPath` (the binary path) and `starlark.lspArguments` (the arguments to that binary). These are available in the VSCode extension settings UI. +If using another binary, the settings to be aware of are `starlark.lspPath` (the +binary path) and `starlark.lspArguments` (the arguments to that binary). These +are available in the VSCode extension settings UI. Based on a combination of: -* Tutorial at https://code.visualstudio.com/api/language-extensions/language-server-extension-guide -* Code for the tutorial at https://github.com/microsoft/vscode-extension-samples/tree/master/lsp-sample -* Syntax files from https://github.com/phgn0/vscode-starlark (which are the Microsoft Python ones with minor tweaks) +- Tutorial at + https://code.visualstudio.com/api/language-extensions/language-server-extension-guide +- Code for the tutorial at + https://github.com/microsoft/vscode-extension-samples/tree/master/lsp-sample +- Syntax files from https://github.com/phgn0/vscode-starlark (which are the + Microsoft Python ones with minor tweaks) ## Pre-requisites -You need to have npm v7+ installed. Afterwards, run `npm install` in this folder and in `client`. +You need to have npm v7+ installed. Afterwards, run `npm install` in this folder +and in `client`. ## Debugging @@ -28,16 +37,22 @@ You need to have npm v7+ installed. Afterwards, run `npm install` in this folder - Follow steps in Pre-requisites section. - Run `npm install vsce` - Run `npm exec vsce package` -- In VS Code, go to Extensions, click on the "..." button in the Extensions bar, select "Install from VSIX" and then select the `starlark-1.0.0.vsix` file that was produced. -- Build the starlark binary with `cargo build --bin=starlark` and then do one of: - - Put it on your `$PATH`, e.g. `cp $CARGO_TARGET_DIR/debug/starlark ~/.cargo/bin/starlark`. - - Configure the setting `starlark.lspPath` for this extension to point to the starlark binary. e.g. `$CARGO_TARGET_DIR/debug/starlark`. +- In VS Code, go to Extensions, click on the "..." button in the Extensions bar, + select "Install from VSIX" and then select the `starlark-1.0.0.vsix` file that + was produced. +- Build the starlark binary with `cargo build --bin=starlark` and then do one + of: + - Put it on your `$PATH`, e.g. + `cp $CARGO_TARGET_DIR/debug/starlark ~/.cargo/bin/starlark`. + - Configure the setting `starlark.lspPath` for this extension to point to the + starlark binary. e.g. `$CARGO_TARGET_DIR/debug/starlark`. ## Updating -Every few months security advisories will arrive about pinned versions of packages. +Every few months security advisories will arrive about pinned versions of +packages. -* `npm audit` to see which packages have security updates. -* `npm audit fix` to fix those issues. -* Try `npm audit`, if it still has issues run `npm update`. -* `npm exec vsce package` to confirm everything still works. +- `npm audit` to see which packages have security updates. +- `npm audit fix` to fix those issues. +- Try `npm audit`, if it still has issues run `npm update`. +- `npm exec vsce package` to confirm everything still works. diff --git a/superconsole/README.md b/superconsole/README.md index e1474515d0790..54db6dc571cfe 100644 --- a/superconsole/README.md +++ b/superconsole/README.md @@ -1,14 +1,25 @@ # A component-based framework for building Rust Text-based User Interfaces (TUIs) -There are several copies of this repo on GitHub, [facebookincubator/superconsole](https://github.com/facebookincubator/superconsole) is the canonical one. +There are several copies of this repo on GitHub, +[facebookincubator/superconsole](https://github.com/facebookincubator/superconsole) +is the canonical one. -The superconsole framework provides a powerful line based abstraction over text based rendering to the terminal. It also provides basic building blocks like line manipulation, and a higher level of composable components. A base set of "batteries" components are included to help developers create Text-based User Interfaces (TUIs) as quickly as possible. +The superconsole framework provides a powerful line based abstraction over text +based rendering to the terminal. It also provides basic building blocks like +line manipulation, and a higher level of composable components. A base set of +"batteries" components are included to help developers create Text-based User +Interfaces (TUIs) as quickly as possible. -The design choices that underly superconsole are selected to prioritize testability, ease of composition, and flexibility. +The design choices that underly superconsole are selected to prioritize +testability, ease of composition, and flexibility. -Superconsole also offers stylization, including italics, underlining, bolding, and coloring text. Furthermore, relying on crossterm ensures that it is compatible with Windows, Unix, and MacOS. +Superconsole also offers stylization, including italics, underlining, bolding, +and coloring text. Furthermore, relying on crossterm ensures that it is +compatible with Windows, Unix, and MacOS. -Finally, superconsole delineates between rendering logic and program state - each render call accepts an immutable reference to state, which components may use to inject state into their otherwise immutable rendering logic. +Finally, superconsole delineates between rendering logic and program state - +each render call accepts an immutable reference to state, which components may +use to inject state into their otherwise immutable rendering logic. ## Demo @@ -46,4 +57,5 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out. ## License -Superconsole is both MIT and Apache License, Version 2.0 licensed, as found in the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. +Superconsole is both MIT and Apache License, Version 2.0 licensed, as found in +the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) files. diff --git a/superconsole/oss/CHANGELOG.md b/superconsole/oss/CHANGELOG.md index 181ce2114362c..1a9232718f385 100644 --- a/superconsole/oss/CHANGELOG.md +++ b/superconsole/oss/CHANGELOG.md @@ -2,13 +2,16 @@ ## 0.2.0 (June 5, 2023) -The major change from the last release is that `State` has been deleted in favor of `render` taking a reference to the components. In addition there were a number of API adjustments, including: +The major change from the last release is that `State` has been deleted in favor +of `render` taking a reference to the components. In addition there were a +number of API adjustments, including: -* Add the type `Lines` wrapping `Vec`. -* Add `Span::new_colored` and `Span::new_colored_lossy`. -* Remove the `line!` macro and add more utilities to the `Line` type, making its internals private. -* Rename `x`/`y` to `width`/`height` where it makes sense. +- Add the type `Lines` wrapping `Vec`. +- Add `Span::new_colored` and `Span::new_colored_lossy`. +- Remove the `line!` macro and add more utilities to the `Line` type, making its + internals private. +- Rename `x`/`y` to `width`/`height` where it makes sense. ## 0.1.0 (February 3, 2022) -* Initial version. +- Initial version. diff --git a/superconsole/oss/CONTRIBUTING.md b/superconsole/oss/CONTRIBUTING.md index 495aa3d935eb3..361b7c7099ed5 100644 --- a/superconsole/oss/CONTRIBUTING.md +++ b/superconsole/oss/CONTRIBUTING.md @@ -1,12 +1,13 @@ # Contributing to Superconsole -We want to make contributing to this project as easy and transparent as possible. +We want to make contributing to this project as easy and transparent as +possible. ## Our Development Process -Superconsole is currently developed in Facebook's internal repositories and then exported -out to GitHub by a Facebook team member; however, we invite you to submit pull -requests as described below. +Superconsole is currently developed in Facebook's internal repositories and then +exported out to GitHub by a Facebook team member; however, we invite you to +submit pull requests as described below. ## Pull Requests @@ -42,5 +43,6 @@ Follow the automatic `rust fmt` configuration. ## License By contributing to Superconsole, you agree that your contributions will be -licensed under both the [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) -files in the root directory of this source tree. +licensed under both the [LICENSE-MIT](LICENSE-MIT) and +[LICENSE-APACHE](LICENSE-APACHE) files in the root directory of this source +tree. diff --git a/superconsole/src/output.rs b/superconsole/src/output.rs index 21dce4201008f..4177f182809bf 100644 --- a/superconsole/src/output.rs +++ b/superconsole/src/output.rs @@ -183,7 +183,7 @@ impl SuperConsoleOutput for NonBlockingSuperConsoleOutput { } #[cfg(test)] -mod test { +mod tests { use crossbeam_channel::Receiver; use super::*; diff --git a/test.py b/test.py index c64dfb79c7e20..cd67ce010936c 100755 --- a/test.py +++ b/test.py @@ -124,11 +124,11 @@ def list_starlark_files(git: bool): ] excludes = [ "starlark-rust/starlark/testcases/", - "tests/meta_only/e2e/test_starlark_data/bad_warning.bzl", - "tests/meta_only/e2e/test_lsp_data/bad_syntax.bzl", - "tests/meta_only/e2e/test_lsp_data/query.bxl", - "tests/meta_only/e2e/test_lsp_data/globals.bzl", - "tests/meta_only/e2e/test_lsp_data/cell/sub/defs.bzl", + "tests/e2e/test_starlark_data/bad_warning.bzl", + "tests/e2e/test_lsp_data/bad_syntax.bzl", + "tests/e2e/test_lsp_data/query.bxl", + "tests/e2e/test_lsp_data/globals.bzl", + "tests/e2e/test_lsp_data/cell/sub/defs.bzl", "**.rs", "**.fixture", "**.buckconfig", @@ -230,17 +230,10 @@ def rustfmt(buck2_dir: Path, ci: bool, git: bool) -> None: "clippy::unwrap-or-default", # Defaults aren't always more clear as it removes the type information when reading code "clippy::enum-variant-names", # Sometimes you do want the same prefixes "clippy::needless_update", # Our RE structs have slightly different definitions in internal and OSS. - "clippy::almost-swapped", # Triggered by Clap v3, perhaps remove when we move to v4 - "clippy::format_collect", # FIXME new in Rust 1.73 - "clippy::needless_pass_by_ref_mut", # FIXME new in Rust 1.73 - "clippy::needless_return", # FIXME new in Rust 1.73 - "clippy::redundant_closure", # FIXME new in Rust 1.73 - "clippy::redundant_locals", # FIXME new in Rust 1.73 + "clippy::needless_pass_by_ref_mut", # Mostly identifies cases where we are accepting `&mut T` because we logically accept a mut reference but don't technically require it (i.e. we want the api to enforce the caller has a mut ref, but we don't technically need it). + "clippy::non_canonical_partial_ord_impl", # Almost exclusively identifies cases where a type delegates ord/partial ord to something else (including Derivative-derived PartialOrd) and in that case being explicit about that delegation is better than following some canonical partialord impl. "clippy::await_holding_lock", # FIXME new in Rust 1.74 "clippy::needless_borrows_for_generic_args", # FIXME new in Rust 1.74 - "clippy::non_canonical_partial_ord_impl", # FIXME new in Rust 1.74 - "clippy::redundant_as_str", # FIXME new in Rust 1.74 - "clippy::unnecessary_map_on_constructor", # FIXME new in Rust 1.74 ] CLIPPY_DENY = [ diff --git a/website/README.md b/website/README.md index 2aee58ee8c97f..99ce4a1a7bd9d 100644 --- a/website/README.md +++ b/website/README.md @@ -1,6 +1,7 @@ # Website -This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. +This website is built using [Docusaurus 2](https://docusaurus.io/), a modern +static website generator. ## Installation @@ -9,28 +10,33 @@ $ yarn global add node-gyp $ yarn ``` -If on Eden you might get faster builds by doing `eden redirect add $PWD/node_modules bind` first. +If on Eden you might get faster builds by doing +`eden redirect add $PWD/node_modules bind` first. ## Build -To build a copy of the static content against the version of `buck2` on your path: +To build a copy of the static content against the version of `buck2` on your +path: ```shell $ yarn build ``` -To build a copy of the static content using `../.buck2.sh` (which builds buck2 from the repo before invoking it): +To build a copy of the static content using `../.buck2.sh` (which builds buck2 +from the repo before invoking it): ```shell $ yarn build_local ``` To build a copy of the static content using Cargo to build buck2: + ```shell $ yarn build_cargo ``` -All of these commands generate static content into the `build` directory and can be served using any static contents hosting service. +All of these commands generate static content into the `build` directory and can +be served using any static contents hosting service. ## Local Development @@ -38,13 +44,18 @@ All of these commands generate static content into the `build` directory and can $ yarn start ``` -This command starts a local development server and opens up a browser window. Any changes to generated Starlark API documentation require running the build command above, but changes to the .md files that are checked into the repository should be reflected live without having to restart the server. +This command starts a local development server and opens up a browser window. +Any changes to generated Starlark API documentation require running the build +command above, but changes to the .md files that are checked into the repository +should be reflected live without having to restart the server. ### Run on devserver -If developing on a devserver, you'll need to create a tunnel from your Mac to the server, so you can access it in the browser. +If developing on a devserver, you'll need to create a tunnel from your Mac to +the server, so you can access it in the browser. To do that, run the following **from your mac**: + ``` ssh -L 3000:localhost:3000 $DEVSERVER ``` @@ -64,4 +75,5 @@ $ yarn start-fb $ GIT_USER= USE_SSH=true yarn deploy ``` -If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. +If you are using GitHub pages for hosting, this command is a convenient way to +build the website and push to the `gh-pages` branch. diff --git a/website/package.json b/website/package.json index 69d6b09d2b3e0..f97407ae8d762 100644 --- a/website/package.json +++ b/website/package.json @@ -45,6 +45,7 @@ }, "resolutions": { "shelljs": "^0.8.5", - "ansi-html": "0.0.8" + "ansi-html": "0.0.8", + "@babel/traverse": "^7.23.2" } } diff --git a/website/sidebars.js b/website/sidebars.js index 4615ee48bd307..6bcc6d309cebc 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -40,7 +40,8 @@ const manualSidebar = [ label: 'About Buck2', items: [ 'why', - 'getting_started', + // The getting_started page is for OSS only. + isInternal() ? [] : 'getting_started', { type: 'category', label: 'Benefits', @@ -57,8 +58,13 @@ const manualSidebar = [ type: 'category', label: 'Concepts', items: [ + 'concepts/key_concepts', 'concepts/concept_map', + 'concepts/build_rule', + 'concepts/build_file', + 'concepts/build_target', 'concepts/target_pattern', + 'concepts/buck_out', 'concepts/visibility', 'concepts/daemon', 'concepts/buckconfig', @@ -78,6 +84,7 @@ const manualSidebar = [ { type: 'autogenerated', dirName: 'users/commands'}, ], }, + 'users/cheat_sheet', { type: 'category', label: 'Troubleshooting', @@ -87,7 +94,10 @@ const manualSidebar = [ isInternal() ? 'users/faq/meta_issues' : [], isInternal() ? 'users/faq/meta_installation' : [], isInternal() ? 'users/faq/remote_execution' : [], - isInternal() ? 'users/faq/starlark_peak_mem' : [], + 'users/faq/starlark_peak_mem', + 'users/faq/buck_hanging', + isInternal() ? 'users/faq/how_to_bisect' : [], + isInternal() ? 'users/faq/how_to_expedite_fix' : [], ], }, { @@ -140,6 +150,8 @@ const manualSidebar = [ 'rule_authors/alias', 'rule_authors/local_resources', 'rule_authors/package_files', + isInternal() ? 'rule_authors/client_metadata' : [], + isInternal() ? 'rule_authors/action_error_handler' : [], { type: 'autogenerated', dirName: 'rule_authors' }, ], }, diff --git a/website/static/img/packages-1.png b/website/static/img/packages-1.png new file mode 100644 index 0000000000000..4f48eb1d97a08 Binary files /dev/null and b/website/static/img/packages-1.png differ