diff --git a/#qa.md# b/#qa.md# new file mode 100644 index 00000000..7072180c --- /dev/null +++ b/#qa.md# @@ -0,0 +1,40 @@ +last issue +bash-5.3$ nix build +trace: Rust 1.92.0-nightly-2025-10-16: +Pre-aggregated package `rust` is not encouraged for stable channel since it contains almost all and uncertain components. +Consider use `default` profile like `rust-bin.stable.latest.default` and override it with extensions you need. +See README for more information. + +this derivation will be built: + /nix/store/jjy833sc0z7xcl495sfkyx2rcqyfigmi-rust-solana-tools-v1.51.drv +building '/nix/store/jjy833sc0z7xcl495sfkyx2rcqyfigmi-rust-solana-tools-v1.51.drv'... +error: builder for '/nix/store/jjy833sc0z7xcl495sfkyx2rcqyfigmi-rust-solana-tools-v1.51.drv' failed with exit code 1; + last 25 log lines: + > Compiling filetime v0.2.25 + > Compiling cpufeatures v0.2.14 + > Compiling itoa v1.0.11 + > Compiling ryu v1.0.18 + > Compiling bootstrap v0.0.0 (/tmp/nix-shell.R9IS5s/nix-shell.TZl15H/nix-build-rust-solana-tools-v1.51.drv-0/k7wrn478pqvwbzcr7gkbjghcphp62kxd-source/src/bootstrap) + > Compiling tar v0.4.42 + > Compiling sha2 v0.10.8 + > Compiling clap_derive v4.5.18 + > Compiling serde_derive v1.0.210 + > Compiling ignore v0.4.23 + > Compiling opener v0.5.2 + > Compiling fd-lock v4.0.2 + > Compiling toml v0.5.11 + > Compiling cmake v0.1.48 + > Compiling object v0.36.4 + > Compiling home v0.5.9 + > Compiling termcolor v1.4.1 + > Compiling clap v4.5.18 + > Compiling clap_complete v4.5.29 + > Compiling build_helper v0.1.0 (/tmp/nix-shell.R9IS5s/nix-shell.TZl15H/nix-build-rust-solana-tools-v1.51.drv-0/k7wrn478pqvwbzcr7gkbjghcphp62kxd-source/src/build_helper) + > Compiling xz2 v0.1.7 + > Finished `dev` profile [unoptimized] target(s) in 1m 55s + > DEBUG: Entering run function, about to execute command. + > ERROR: Failed to parse 'config.toml': unknown field `CARGO_HOME` + > Build completed unsuccessfully in 0:01:55 + For full logs, run: + nix log /nix/store/jjy833sc0z7xcl495sfkyx2rcqyfigmi-rust-solana-tools-v1.51.drv +bash-5.3$ \ No newline at end of file diff --git a/.gemini/commit-message.txt b/.gemini/commit-message.txt new file mode 100644 index 00000000..c5a6cc7f --- /dev/null +++ b/.gemini/commit-message.txt @@ -0,0 +1,10 @@ +feat: Centralize git command helpers in bootstrap-config-builder + +Created `bootstrap-config-builder/src/utils/git_command_helpers.rs` to +centralize generic functions for executing git commands via `std::process::Command`. +This includes `run_git_command` and `output_result`, adapted to use `anyhow::Result` +for better error handling. The `bootstrap-config-builder/src/utils/mod.rs` +was updated to include this new module. + +This change aims to reduce potential code duplication and provide a single +point of maintenance for shell-based git operations within the `bootstrap-config-builder` crate. \ No newline at end of file diff --git a/.gitignore b/.gitignore index c62ac0f3..ceb873ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,17 @@ /build/ /standalonex/build/ /standalonex/result +result +logs/ +Makefile~ +*.log +*~ +*# +*/build +/standalonex/src/target/ +/standalonex/src/bootstrap/build/ +/standalonex/src/bootstrap/target/ +target/ +*.d +*.so +/.pre-commit-config.local.yamlnix-build-scripts/.#Makefile diff --git a/.pre-commit-config.local.yaml b/.pre-commit-config.local.yaml new file mode 100644 index 00000000..d333ecb4 --- /dev/null +++ b/.pre-commit-config.local.yaml @@ -0,0 +1,36 @@ +# DO NOT MODIFY +# This file was generated by git-hooks.nix +{ + "default_stages": [ + "pre-commit" + ], + "repos": [ + { + "hooks": [ + { + "always_run": false, + "args": [], + "entry": "/nix/store/p7k9clwmbbr98c171269n907r2730wl3-nixpkgs-fmt-1.3.0/bin/nixpkgs-fmt", + "exclude": "^$", + "exclude_types": [], + "fail_fast": false, + "files": ".nix$", + "id": "nixpkgs-fmt", + "language": "system", + "name": "nixpkgs-fmt", + "pass_filenames": true, + "require_serial": false, + "stages": [ + "pre-commit" + ], + "types": [ + "file" + ], + "types_or": [], + "verbose": false + } + ], + "repo": "local" + } + ] +} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4ec0f941 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +# DO NOT MODIFY +# This file was generated by git-hooks.nix +{ + "default_stages": [ + "pre-commit" + ], + "repos": [ + { + "hooks": [ + { + "always_run": false, + "args": [], + "entry": "/nix/store/p7k9clwmbbr98c171269n907r2730wl3-nixpkgs-fmt-1.3.0/bin/nixpkgs-fmt", + "exclude": "^$", + "exclude_types": [], + "fail_fast": false, + "files": "\\.nix$", + "id": "nixpkgs-fmt", + "language": "system", + "name": "nixpkgs-fmt", + "pass_filenames": true, + "require_serial": false, + "stages": [ + "pre-commit" + ], + "types": [ + "file" + ], + "types_or": [], + "verbose": false + }, + { + "always_run": false, + "args": [], + "entry": "/nix/store/grb48jw3fbszgxvhjqm7n5ckhmgk6inw-typos-1.35.6/bin/typos --config /nix/store/wvhry1k8b04i0hi7ir489kh08sd8ll5i-typos-config.toml --force-exclude", + "exclude": "^$", + "exclude_types": [], + "fail_fast": false, + "files": "", + "id": "typos", + "language": "system", + "name": "typos", + "pass_filenames": true, + "require_serial": false, + "stages": [ + "pre-commit" + ], + "types": [ + "text" + ], + "types_or": [], + "verbose": false + } + ], + "repo": "local" + } + ] +} diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..7b58003e --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,693 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bootstrap-config-builder" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "serde", + "serde_json", + "tempfile", + "toml 0.8.23", +] + +[[package]] +name = "bootstrap-config-processor" +version = "0.1.0" +dependencies = [ + "bootstrap-config-utils", +] + +[[package]] +name = "bootstrap-config-utils" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", + "stage0_parser_crate", + "toml 0.5.11", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.5.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "config_core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "config_macros" +version = "0.1.0" +dependencies = [ + "config_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "config_tests" +version = "0.1.0" +dependencies = [ + "config_core", + "config_macros", +] + +[[package]] +name = "configuration-nix" +version = "0.1.0" +dependencies = [ + "clap", +] + +[[package]] +name = "env_filter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "flake-template-generator" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "toml 0.8.23", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jiff" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "stage0_parser_crate" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..fcea9516 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = [ + "standalonex/src/bootstrap/src/core/config_utils", + "standalonex/src/bootstrap/src/core/config_processor", + "standalonex/src/stage0_parser_crate", + "configuration-nix", + "bootstrap-config-builder", + "standalonex/src/bootstrap/src/core/config_crates/config_tests", "flake-template-generator", +] diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 00000000..8afd5342 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,54 @@ +## Gemini Added Memories +- For Nix flake inputs, always use the pattern `github:meta-introspector/(repo name)?ref=(branch name)&dir=(path within repo)`. The current repo is `time-2025`, and the current branch is `feature/lattice-30030-homedir`, but these can vary. The `dir` parameter should be inserted as needed. +- nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; +- use git commit -F commit-message.txt instead of long commands +- In the 'nix2make2nix' system, each make target, each nix flake, and each content are all represented by emoji strings or primes. +- self.url ="github:meta-introspector/time-2025?ref=feature/aimyc-001-cleanbench" +- The user has an idea to create a continuation in the Nix REPL to drive LLM jobs from inside a Nix shell, allowing the LLM to generate and reflect on Nix expressions. +- To use the local flake registry, set the NIX_REGISTRY_CONFIG environment variable to the absolute path of the registry.json file: export NIX_REGISTRY_CONFIG=/data/data/com.termux.nix/files/home/pick-up-nix2/source/github/meta-introspector/streamofrandom/2025/registry.json +- In the context of AI Life Mycology, graphs are considered to be the 'knots' or 'Quasi-Fibers'. +- The project aims for an 8-fold recursion system: 8 stages of self-hosting Rust compilation, each improving the next, culminating in a smart contract system where Rust functions and the compiler are translated to blockchain data. + +### Bootstrap Configuration Builder Updates + +This section details recent developments in configuring the Rust bootstrap process, particularly focusing on the `bootstrap-config-builder` crate and its integration with Nix. + +**Goal:** The primary objective is to enable the generation of `config.toml` files for the Rust bootstrap process, leveraging Nix store paths for `rustc` and `cargo`, and to facilitate systematic testing of different Rust toolchain versions from the Nix store. This is a foundational step towards an 8-fold recursion system for eBPF Rust bootstrapping. + +**Key Changes and Learnings:** + +1. **`bootstrap-config-builder` Refactoring:** + * The `bootstrap-config-builder` crate was refactored to consolidate its `main` function into `src/lib.rs`, making it a library that can also be run as a binary. The redundant `src/main.rs` file was removed. + * A new binary target, `bootstrap-config-generator`, was added to `bootstrap-config-builder/Cargo.toml` to explicitly run the `main` function from `src/lib.rs`. This allows for execution via `cargo run --bin bootstrap-config-generator`. + * A type mismatch for the `dry_run` argument in `bootstrap-config-builder/src/config.rs` was resolved, ensuring correct handling of boolean flags. + * The `example.toml` template content was inlined directly into `bootstrap-config-builder/src/utils/format_file.rs`. This eliminates a file system dependency and potential path resolution issues, making the configuration generation process more robust. + * The `template_path` argument was removed from `bootstrap-config-builder/src/utils/construct_config_content.rs` as it is no longer needed. + +2. **Nix Integration for Rust Toolchain Paths:** + * We successfully obtained the Nix store paths for `rustc` and `cargo` using `nix-shell` and `which` commands. These paths are crucial for ensuring the Rust bootstrap process uses precisely defined and versioned compilers from the Nix store. + * `rustc`: `/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc` + * `cargo`: `/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo` + * The `bootstrap-config-generator` can now be invoked with `--rustc-path` and `--cargo-path` arguments to inject these Nix store paths into the generated `config.toml`. + +3. **Rust Source Flake Path Handling:** + * The `bootstrap-config-generator` expects a *local path* to the Rust source flake (e.g., `/data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src`) via the `--rust-src-flake-path` argument. It validates this path by checking for the presence of `src/ci/channel` within it. + * This local path is essential for the Rust bootstrap process to access the compiler's source code (sysroot). + +**How to Generate `config.toml`:** + +To generate a `config.toml` with specific `rustc` and `cargo` paths, and a local Rust source path, navigate to the `bootstrap-config-builder` directory and run: + +```bash +cargo run --bin bootstrap-config-generator -- \ + --rustc-path /nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc \ + --cargo-path /nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo \ + --project-root /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-bootstrap-nix \ + --rust-src-flake-path /data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src \ + --output generated_config.toml +``` + +This command will create a `generated_config.toml` file in the `bootstrap-config-builder` directory, containing the specified paths and other default configuration values. + +**Next Steps:** + +The generated `config.toml` now needs to be integrated into the actual Rust bootstrap process. This involves understanding how the main `rust-bootstrap-nix` project consumes its `config.toml` and adapting it to use the newly generated configuration. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..7e1f8b1b --- /dev/null +++ b/Makefile @@ -0,0 +1,69 @@ +.PHONY: all build fast-build run-config-builder-dry-run build-config-builder generate-seed-config generate-flake-dir + +all: build build-config-builder + +build: generate-config + $(MAKE) -C nix-build-scripts/ + +fast-build: + $(MAKE) -C nix-build-scripts/ fast-build + +build-config-builder: + @echo "Building bootstrap-config-builder..." + cargo build --package bootstrap-config-builder + +generate-config: build-config-builder + @echo "Generating config.toml using bootstrap-config-builder..." + $(eval NIXPKGS_PATH := $(shell nix build .#nixpkgsOutPath --no-link --print-out-paths)) + $(eval RUST_OVERLAY_PATH := $(shell nix build .#rustOverlayOutPath --no-link --print-out-paths)) + $(eval RUST_BOOTSTRAP_NIX_PATH := $(shell nix build .#rustBootstrapNixOutPath --no-link --print-out-paths)) + $(eval CONFIGURATION_NIX_PATH := $(shell nix build .#configurationNixOutPath --no-link --print-out-paths)) + $(eval RUST_SRC_FLAKE_PATH := $(shell nix build .#rustSrcFlakeOutPath --no-link --print-out-paths)) + @RUST_LOG=debug ./target/debug/bootstrap-config-builder 0 aarch64-unknown-linux-gnu \ + --project-root $(CURDIR) \ + --system aarch64-linux \ + --output config.toml \ + --rust-bootstrap-nix-flake-ref "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify" \ + --rust-src-flake-ref "github:meta-introspector/rust?ref=feature/CRQ-016-nixify" \ + --nixpkgs-path $(NIXPKGS_PATH) \ + --rust-overlay-path $(RUST_OVERLAY_PATH) \ + --rust-bootstrap-nix-path $(RUST_BOOTSTRAP_NIX_PATH) \ + --configuration-nix-path $(CONFIGURATION_NIX_PATH) \ + --rust-src-flake-path $(RUST_SRC_FLAKE_PATH) + +generate-seed-config: build-config-builder + @echo "Generating seed config.toml using bootstrap-config-generator..." + cargo run --bin bootstrap-config-generator -- \ + --output bootstrap-config-builder/generated_config.toml \ + --project-root $(CURDIR) \ + --rust-src-flake-path /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src + +generate-flake-dir: + $(MAKE) -C flake-template-generator generate-flake + +generate-seed-config: build-config-builder + @echo "Generating seed config.toml using bootstrap-config-generator..." + cargo run --bin bootstrap-config-generator -- \ + --output bootstrap-config-builder/generated_config.toml \ + --project-root $(CURDIR) \ + --rust-src-flake-path /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src + +run-config-builder-dry-run: + @echo "Running bootstrap-config-builder in dry-run mode..." + $(eval NIXPKGS_PATH := $(shell nix build .#nixpkgsOutPath --no-link --print-out-paths)) + $(eval RUST_OVERLAY_PATH := $(shell nix build .#rustOverlayOutPath --no-link --print-out-paths)) + $(eval RUST_BOOTSTRAP_NIX_PATH := $(shell nix build .#rustBootstrapNixOutPath --no-link --print-out-paths)) + $(eval CONFIGURATION_NIX_PATH := $(shell nix build .#configurationNixOutPath --no-link --print-out-paths)) + $(eval RUST_SRC_FLAKE_PATH := $(shell nix build .#rustSrcFlakeOutPath --no-link --print-out-paths)) + @RUST_LOG=debug ./target/debug/bootstrap-config-builder 0 aarch64-unknown-linux-gnu \ + --project-root $(CURDIR) \ + --system aarch64-linux \ + --output generated_config.toml \ + --rust-bootstrap-nix-flake-ref "github:meta-introspector/rust-bootstrap-nix?rev=e1215ab7f9aa7674c57155c59bfc6ed2c1d10e14" \ + --rust-src-flake-ref "github:meta-introspector/rust?rev=e6c1b92d0abaa3f64032d6662cbcde980c826ff2" \ + --nixpkgs-path $(NIXPKGS_PATH) \ + --rust-overlay-path $(RUST_OVERLAY_PATH) \ + --rust-bootstrap-nix-path $(RUST_BOOTSTRAP_NIX_PATH) \ + --configuration-nix-path $(CONFIGURATION_NIX_PATH) \ + --rust-src-flake-path $(RUST_SRC_FLAKE_PATH) \ + --dry-run diff --git a/README.md b/README.md new file mode 100644 index 00000000..456cc819 --- /dev/null +++ b/README.md @@ -0,0 +1,505 @@ +# rust-bootstrap-nix + +This repository provides a Nix-based development and build environment for Rust projects, with a focus on integrating `sccache` for accelerated compilation and managing the `x.py` build system. It includes various Nix flakes for environment setup, JSON output processing, and build command evaluation, alongside shell scripts for debugging, development, and testing. + +## Key Features + +* **Reproducible Development Environments:** Utilizes Nix flakes to define consistent Python and Rust development shells. +* **`sccache` Integration:** Accelerates Rust compilation through `sccache` caching. +* **`x.py` Build System Support:** Provides tools and environments for working with the `x.py` build orchestration script. +* **JSON Output Processing:** Includes flakes for capturing and analyzing JSON metadata generated by the build process. + +## Dynamic Nix-based Configuration + +This project now features a dynamic, Nix-based configuration system that allows for precise control over the Rust bootstrap process. This system is designed to create a "lattice of Nix flakes," enabling reproducible builds and making it easy to experiment with different versions of Rust and its dependencies. + +### How it Works + +The core of this system is the `generated_config.toml` file, which is generated by the `bootstrap-config-builder` utility. This file contains the exact Nix store paths for all the tools and dependencies required for the build, including `rustc`, `cargo`, `nixpkgs`, `rust-overlay`, and the Rust source code itself. + +The Rust bootstrap process has been modified to read this `generated_config.toml` file and use the paths within it to configure the build. If the `generated_config.toml` file is not present, the bootstrap process will dynamically fetch the required Nix store paths using `nix flake prefetch` and `nix path-info`. + +### Usage + +To use this new system, you can either: + +1. **Generate `generated_config.toml` manually:** Run the `bootstrap-config-builder` with the desired `--rustc-path`, `--cargo-path`, and `--rust-src-flake-path` arguments. + * **Using Makefile target:** You can use the `generate-seed-config` Makefile target to generate the `generated_config.toml` in the `bootstrap-config-builder/` directory. + ```bash + make generate-seed-config + ``` +2. **Generate the flake directory:** Use the `generate-flake-dir` Makefile target to create the `target/generated-flake` directory containing the dynamically generated `flake.nix` and `config.toml`. + ```bash + make generate-flake-dir + ``` +3. **Let the bootstrap process handle it:** If `generated_config.toml` is not present, the bootstrap process will automatically resolve the Nix paths for you. + +For more detailed information, please refer to the `docs/Nix_Integration.md` file. + +## Building the Standalone Bootstrap + +To build the standalone Rust bootstrap environment, which is particularly useful for "Nix on Droid" (aarch64-linux) environments, use the following Nix command: + +```bash +nix build ./standalonex#packages.aarch64-linux.default +``` + +This command will build the default package defined within the `standalonex/flake.nix` for the `aarch64-linux` architecture. + +## Repository Overview + +# Repository Overview: `rust-bootstrap-nix` + +This repository serves as a comprehensive Nix-based environment for developing, building, and testing Rust projects, with a particular focus on integrating `sccache` for build acceleration and leveraging a custom `x.py` build orchestration system. It is designed to provide reproducible build environments across different architectures (`aarch64-linux` and `x86_64-linux`). + +## Core Purpose + +The primary goal of `rust-bootstrap-nix` is to streamline the Rust development workflow within a Nix ecosystem. This involves: + +1. **Reproducible Toolchains:** Providing consistent and isolated Rust compiler and Cargo toolchains via Nix flakes. +2. **Build Acceleration:** Integrating `sccache` to significantly speed up Rust compilation times. +3. **Custom Build Orchestration:** Utilizing a Python-based `x.py` script for managing complex build processes, including dependency handling and build step execution. +4. **Build Metadata Extraction:** Generating and processing structured JSON output from the build process for analysis and further automation. +5. **Modular Flake Structure:** Breaking down the environment and build logic into smaller, interconnected Nix flakes for better organization and reusability. + +## Key Components + +The repository is structured around several key components: + +* **Nix Flakes:** A collection of `flake.nix` files that define development environments, packages, and build logic. These include the root flake, sub-flakes for JSON processing, Rust evaluation, and a standalone `x.py` environment. +* **Shell Scripts:** Various `.sh` scripts for common tasks such as entering development shells, debugging builds, diagnosing environment issues, and updating flakes. +* **Configuration Files:** `config.toml` files that specify build settings, toolchain paths, and vendoring options. +* **`standalonex/` Directory:** A critical component containing the `x.py` build orchestration script, Python utilities (`test_json_output.py`, `wrap_rust.py`), and Rust source code (`src/`). This directory is central to how the Rust project is built and how build metadata is generated. +* **`src/bootstrap/bootstrap.py`:** The core Python script within `standalonex/src/bootstrap/` that implements the detailed logic for the Rust build process, including toolchain management, environment setup, and JSON output generation. + +## How it Works (High-Level) + +The system leverages Nix flakes to define a hermetic build environment. The root `flake.nix` sets up a development shell with Python, Rust, and `sccache`. The `x.py` script (located in `standalonex/`) acts as the primary interface for building the Rust project. During the build, `x.py` (specifically through its `bootstrap` module) can generate JSON output containing detailed information about the compilation steps. Other flakes then consume and process this JSON data, enabling advanced analysis and automation of the Rust build process. + +## Configuration Documentation + +# Configuration Documentation + +This document details the various configuration files used within the `rust-bootstrap-nix` repository, primarily focusing on `config.toml` files that influence the Rust build process and environment setup. + +## 1. Root `config.toml` + +**File Path:** `/config.toml` + +**Description:** This is the primary configuration file for the overall `rust-bootstrap-nix` environment. It explicitly defines how the Rust toolchain is sourced and how the build environment is isolated. + +**Key Settings:** + +* `vendor = true`: + * **Purpose:** Enables vendoring for the Rust build process. This means that dependencies are expected to be present locally (e.g., in a `vendor/` directory) rather than being downloaded from the internet during the build. This is crucial for reproducible builds in a Nix environment. +* `rustc = "/nix/store/.../bin/rustc"`: + * **Purpose:** Specifies the absolute path to the `rustc` (Rust compiler) executable within the Nix store. This ensures that the build uses a precisely defined and versioned compiler provided by Nix. +* `cargo = "/nix/store/.../bin/cargo"`: + * **Purpose:** Specifies the absolute path to the `cargo` (Rust package manager) executable within the Nix store. Similar to `rustc`, this guarantees the use of a specific, Nix-managed `cargo` instance. +* `HOME = "/data/data/com.termux.nix/files/usr/tmp/..."`: + * **Purpose:** Sets the `HOME` environment variable to a temporary, isolated directory. This prevents the build process from interacting with or polluting the user's actual home directory. +* `CARGO_HOME = "/data/data/com.termux.nix/files/usr/tmp/.../.cargo"`: + * **Purpose:** Sets the `CARGO_HOME` environment variable to a temporary `.cargo` directory. This ensures that Cargo's caches, registries, and other state are kept isolated within the build environment. + +**Overall Purpose:** The root `config.toml` is fundamental for establishing a hermetic and reproducible Rust build environment. It explicitly directs the build system to use Nix-provided tools and to operate within a clean, temporary workspace. + +## 2. `standalonex/config.toml` + +**File Path:** `/standalonex/config.toml` + +**Description:** This configuration file is specific to the `standalonex` component, which is a standalone environment for the `x.py` build system. It defines the Rust toolchain paths that `x.py` should use within this isolated context. + +**Key Settings:** + +* `rustc = "/nix/store/.../bin/rustc"`: + * **Purpose:** Similar to the root `config.toml`, this specifies the absolute path to the `rustc` executable, ensuring that the `standalonex` environment uses a Nix-provided compiler. +* `cargo = "/nix/store/.../bin/cargo"`: + * **Purpose:** Specifies the absolute path to the `cargo` executable for the `standalonex` environment, guaranteeing the use of a specific, Nix-managed `cargo` instance. + +**Overall Purpose:** This `config.toml` ensures that the `standalonex` build environment, particularly when running `x.py`, is correctly configured with the appropriate Nix-provided Rust toolchain binaries. + +## 3. `standalonex/config.old.toml` + +**File Path:** `/standalonex/config.old.toml` + +**Description:** This file appears to be an older or template version of `standalonex/config.toml`. It is specifically used by the `standalonex/flake.nix`'s `buildPhase` as a base to generate the active `config.toml` by injecting the correct Nix store paths for `rustc` and `cargo` using `sed`. + +**Purpose:** To serve as a template for generating the runtime `config.toml` within the `standalonex` build process, allowing for dynamic injection of Nix-specific paths. + +## Configuring Relocatable Installation Paths for Nix + +For Nix-based builds and to ensure the resulting artifacts are relocatable, it's crucial to properly configure the installation paths. The `[install]` section in your `config.toml` allows you to define a base prefix for all installed components. + +### `[install]` Section + +This section controls where the built artifacts will be placed. + +* `prefix`: + * **Purpose:** Specifies the base directory for all installed components. In a Nix environment, this will typically be a path within the Nix store (e.g., `/nix/store/...-rust-toolchain`). All other installation paths (like `bindir`, `libdir`, etc.) will be derived from this prefix unless explicitly overridden. + * **Example:** `prefix = "/nix/store/some-hash-my-rust-package"` + +* `bindir`: + * **Purpose:** Specifies the directory for executable binaries. + * **Behavior:** If `prefix` is set and `bindir` is *not* explicitly defined, `bindir` will automatically default to `prefix/bin`. This ensures that your executables are placed correctly within the specified installation prefix. + * **Example (explicitly set):** `bindir = "/usr/local/bin"` (overrides the default `prefix/bin`) + +* `libdir`, `sysconfdir`, `docdir`, `mandir`, `datadir`: + * **Purpose:** These fields specify directories for libraries, configuration files, documentation, manual pages, and data files, respectively. + * **Behavior:** If `prefix` is set, these paths are typically expected to be relative to the `prefix` unless an absolute path is provided. + +### Nix-Specific Binary Patching + +The `[build]` section also includes a relevant option for Nix: + +* `patch-binaries-for-nix`: + * **Purpose:** This boolean option enables Nix-specific patching of binaries. This is essential for ensuring that compiled artifacts are truly relocatable within the Nix store, often involving adjustments to RPATHs and other internal paths. + * **Example:** `patch-binaries-for-nix = true` + +### Example `config.toml` for Relocatable Nix Builds + +```toml +# config.toml +[install] +prefix = "/nix/store/some-hash-my-rust-package" +# bindir will automatically be set to "/nix/store/some-hash-my-rust-package/bin" +# libdir = "lib" # would resolve to /nix/store/some-hash-my-rust-package/lib + +[build] +patch-binaries-for-nix = true +``` + +This configuration ensures that your Rust project builds and installs in a manner compatible with Nix's strict path requirements, promoting reproducibility and relocatability. + +## Preconditions for Nix Flake Build + +The `test_nix_preconditions.sh` script verifies essential environmental setups required for a successful Nix-based build of the Rust bootstrap. Ensuring these preconditions are met helps in maintaining a reproducible and stable build environment. + +### 1. Nix Command Availability + +* **Check:** Verifies that the `nix` command-line tool is installed and accessible in the system's `PATH`. +* **Importance:** Nix is fundamental to this build system, as it manages dependencies, builds packages, and ensures reproducibility. Without the `nix` command, the build process cannot proceed. + +### 2. Rust Toolchain Sysroot Existence + +* **Check:** Evaluates the Nix store path for the `pkgs.rust-bin.stable."1.84.1".default` Rust toolchain (including its source) and confirms that the Rust source directory exists within it. +* **Importance:** The Rust bootstrap process often requires access to the Rust compiler's source code (sysroot) for various build stages and internal operations. This precondition ensures that the necessary source components are available from the Nix-managed Rust toolchain. + +### 3. Rust Source Flake (rustSrcFlake) Existence + +* **Check:** Evaluates the Nix store path for the `rustSrcFlake` input (which represents the Rust compiler's source code) as defined in `standalonex/flake.nix`, and verifies that this path exists and contains a known file (`src/ci/channel`). +* **Importance:** The `bootstrap` binary needs to know the location of the Rust compiler's source tree to perform its build tasks. This precondition ensures that the `rustSrcFlake` input is correctly resolved and available, providing the necessary source for the bootstrap process. + +## Nix Flakes Documentation + +# Nix Flakes Documentation + +## 1. Root `flake.nix` + +**File Path:** `/flake.nix` + +**Description:** This flake defines a Python and Rust development environment, with a strong emphasis on integrating `sccache` for accelerated Rust compilation. It supports both `aarch64-linux` and `x86_64-linux` systems. The core functionality revolves around providing a customized Rust toolchain that leverages `sccache` during the build process, particularly when running `python x.py build`. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * A custom `nixpkgs` instance, likely providing specific package versions or configurations tailored for the `meta-introspector` ecosystem. +* `rust-overlay`: `github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify` + * A custom Nix overlay for Rust, also sourced from `meta-introspector`, suggesting specialized Rust toolchain management. +* `rustSrcFlake`: `github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2` + * Points to a specific commit of a `rust` repository within `meta-introspector` organization. This appears to be the foundational Rust source that this flake extends and builds upon. + +**Outputs:** + +* **`devShells..default` (for `aarch64-linux` and `x86_64-linux`):** + * Provides a comprehensive development environment. + * **Packages Included:** + * `rustToolchain` (nightly channel, with specific targets configured) + * `python3` + * `python3Packages.pip` + * `git` + * `curl` + * `which` + * **`shellHook`:** Sets `HOME` and `CARGO_HOME` to `$TMPDIR/.cargo` respectively, ensuring a clean and isolated build environment within the shell. + * **`nativeBuildInputs`:** `binutils`, `cmake`, `ninja`, `pkg-config`, `nix`. These are tools required during the build phase. + * **`buildInputs`:** `openssl`, `glibc.out`, `glibc.static`. These are runtime dependencies. + * **Environment Variables:** `RUSTC_ICE` is set to "0", and `LD_LIBRARY_PATH` is configured. + +* **`sccachedRustc` Function:** + * A local function that takes `system`, `pkgs`, and `rustToolchain` as arguments. + * Its primary role is to wrap the `rustSrcFlake`'s default package with `sccache` capabilities. + * **Modifications:** + * Adds `pkgs.sccache` and `pkgs.curl` to `nativeBuildInputs`. + * **`preConfigure`:** Injects environment variables (`RUSTC_WRAPPER`, `SCCACHE_DIR`, `SCCACHE_TEMPDIR`) to enable `sccache` and starts the `sccache` server. + * **`buildPhase`:** Significantly customizes the build process. It creates a `config.toml` file with `vendor = true`, and sets `rustc` and `cargo` paths to the provided `rustToolchain` binaries. It also sets `HOME` and `CARGO_HOME` for the build and executes `python x.py build`. This indicates that `x.py` is a central build orchestration script. + * **`preBuild` and `postBuild`:** Integrates `sccache` statistics reporting (`sccache --zero-stats`, `sccache --show-stats`, `sccache --stop-server`). + +* **`packages..default` (for `aarch64-linux` and `x86_64-linux`):** + * These outputs provide the `sccache`-enabled Rust compiler package, which is the result of applying the `sccachedRustc` function to the respective system's `rustToolchain`. + +**Overall Purpose:** The root `flake.nix` serves as the entry point for setting up a robust, reproducible, and performance-optimized (via `sccache`) development and build environment for a Rust project that likely uses `python x.py build` as its primary build mechanism. It heavily relies on custom `meta-introspector` Nix inputs for its base components. + +## 2. `flakes/config/flake.nix` + +**File Path:** `/flakes/config/flake.nix` + +**Description:** This flake is designed to read and process JSON output, specifically `xpy_json_output.json`, which is expected to be generated by the `rust-bootstrap-nix` project. It parses this JSON content and makes it available as a Nix package. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustBootstrapNix`: `github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001` + * **Self-Reference:** This input refers to the main `rust-bootstrap-nix` repository itself, specifically pointing to the `feature/bootstrap-001` branch. This establishes a dependency on the outputs of the main project's flake. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output creates a derivation named `processed-json-output`. + * It reads the `xpy_json_output.json` file from the `rustBootstrapNix.packages.aarch64-linux.default` (which is the `sccache`-enabled Rust compiler package from the root flake). + * The content of `xpy_json_output.json` is parsed as JSON using `builtins.fromJSON`. + * The parsed JSON content is then written to `$out/output.txt` within the derivation. + +**Overall Purpose:** This flake acts as a consumer of the `xpy_json_output.json` file produced by the main `rust-bootstrap-nix` build process. It allows for the structured consumption and further processing of this JSON data within the Nix ecosystem. + +## 3. `flakes/evaluate-rust/flake.nix` + +**File Path:** `/flakes/evaluate-rust/flake.nix` + +**Description:** This flake provides a library function `evaluateCommand` designed for recursively evaluating Rust build commands and generating Nix packages. It aims to integrate `naersk` for `cargo build` commands and provides a generic mechanism for other commands. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `naersk`: `github:meta-introspector/naersk?ref=feature/CRQ-016-nixify` + * This input is for `rust2nix` functionality, indicating that this flake intends to use `naersk` to convert Rust projects into Nix derivations. + +**Outputs:** + +* **`lib.evaluateCommand` function:** This is the primary output, a recursive function with the following parameters: + * `commandInfo`: An attribute set containing `command` (the executable, e.g., "cargo", "rustc"), `args` (a list of arguments), and `env` (environment variables). + * `rustSrc`: The source code of the Rust project. + * `currentDepth`: The current recursion depth. + * `maxDepth`: The maximum recursion depth to prevent infinite loops. + + **Function Logic:** + * **Base Case (Recursion Limit):** If `currentDepth` reaches `maxDepth`, it returns a derivation indicating that the recursion limit was reached. + * **`cargo build` Case:** If the command is `cargo` and includes the `build` argument, it uses `naersk.lib.${pkgs.system}.buildPackage` to create a Nix derivation. It passes `cargoBuildFlags` and `env` directly to `naersk`. This is a key integration point for Rust projects. + * **Other Commands Case:** For any other command (e.g., `rustc` directly), it creates a simple `pkgs.runCommand` derivation. It executes the command with its arguments and environment variables, capturing stdout and stderr to `output.txt`. + +**Overall Purpose:** This flake provides a powerful, recursive mechanism to analyze and build Rust projects within Nix. By integrating `naersk`, it can effectively handle `cargo build` commands, transforming them into reproducible Nix derivations. The recursive nature suggests it might be used to trace and build dependencies or stages of a complex Rust build process. + +## 4. `flakes/json-processor/flake.nix` + +**File Path:** `/flakes/json-processor/flake.nix` + +**Description:** This flake defines a Nix package that provides a Python environment with `jq` and `python3` installed. It's intended for processing JSON data, likely in a command-line or scripting context. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. + +**Outputs:** + +* **`packages.aarch64-linux.default` and `packages.x86_64-linux.default`:** + * These outputs define a Nix package for each architecture. + * The package is a `pkgs.mkShell` (which is typically used for development shells, but can also be used to create environments with specific tools). + * **Packages Included:** + * `pkgs.jq`: A lightweight and flexible command-line JSON processor. + * `pkgs.python3`: The Python 3 interpreter. + +**Overall Purpose:** This flake provides a convenient, reproducible environment for working with JSON data using `jq` and Python. It's a utility flake that can be imported by other flakes or used directly to get a shell with these tools. + +## 5. `flakes/json-processor-flake/flake.nix` + +**File Path:** `/flakes/json-processor-flake/flake.nix` + +**Description:** This flake is very similar to `flakes/config/flake.nix` but specifically targets the `standalonex` flake within the `rust-bootstrap-nix` repository. Its purpose is to read and process the `xpy_json_output.json` generated by the `standalonex` flake. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `standalonex`: `github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=standalonex` + * **Self-Reference:** This input directly references the `standalonex` sub-flake within the `rust-bootstrap-nix` repository, specifically pointing to the `feature/bootstrap-001` branch and the `standalonex` directory. This demonstrates how sub-flakes within the same repository can expose their outputs for consumption by other flakes. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output creates a derivation named `processed-json-output`. + * It reads the `xpy_json_output.json` file from the `standalonex.packages.aarch64-linux.default` (which is the default package output of the `standalonex` flake). + * The content of `xpy_json_output.json` is parsed as JSON using `builtins.fromJSON`. + * The parsed JSON content is then written to `$out/output.txt` within the derivation. + +**Overall Purpose:** This flake serves as a dedicated consumer and processor for the JSON output specifically from the `standalonex` component of the `rust-bootstrap-nix` project. It highlights the modularity of Nix flakes, allowing specific parts of a larger project to expose their outputs for consumption by other flakes. + +## 6. `flakes/xpy-json-output-flake/flake.nix` + +**File Path:** `/flakes/xpy-json-output-flake/flake.nix` + +**Description:** This flake is specifically designed to execute the `x.py build --json-output` command from the `rustSrc` input and expose the resulting JSON output directory as a Nix package. This is a crucial flake for understanding the build process and its generated metadata. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustSrc`: `github:meta-introspector/rust?ref=d772ccdfd1905e93362ba045f66dad7e2ccd469b` + * This input points to a specific commit of the `rust` repository within `meta-introspector`. It's marked as `flake = false`, indicating it's treated as a plain source input rather than another Nix flake. This `rustSrc` is where the `x.py` script resides. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output is a derivation named `xpy-json-output-derivation`. + * It uses `pkgs.runCommandLocal` to execute a local command. + * **`nativeBuildInputs`:** Includes `pkgs.python3` because `x.py` is a Python script. + * **`src`:** The `rustSrc` input is used as the source for this derivation. + * **Build Phase:** + * It creates an output directory `$out`. + * It then executes `python3 $src/x.py build --json-output $out`. This command is responsible for running the `x.py` build script and directing its JSON output to the `$out` directory of this derivation. + +**Overall Purpose:** This flake provides a way to capture and expose the structured JSON output generated by the `x.py` build system of the `rustSrc` project. This output likely contains metadata about the build, such as compilation steps, dependencies, or configuration, which can then be consumed and analyzed by other Nix flakes (like the `json-processor` flakes we've seen). + +## 7. `minimal-flake/flake.nix` + +**File Path:** `/minimal-flake/flake.nix` + +**Description:** This flake provides a very basic Python development environment and a simple "hello world" Python script packaged as a Nix derivation. It serves as a minimal example or a starting point for Python-centric Nix flakes. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. + +**Outputs:** + +* **`devShell`:** + * A development shell named `minimal-python-dev-shell`. + * **Packages Included:** `python3` and `git`. This provides a basic environment for Python development and version control. + +* **`packages..helloPython`:** + * A Nix package named `helloPython` for the `aarch64-linux` system. + * It uses `pkgs.writeScriptBin` to create an executable script. + * The script is a simple Python program that prints "Hello from Nix Python!". + +**Overall Purpose:** This flake demonstrates how to set up a minimal Python development environment and package a simple Python script using Nix. It's likely used for quick testing, as a template, or to illustrate basic Nix flake concepts for Python projects. + +## 8. `standalonex/flake.nix` + +**File Path:** `/standalonex/flake.nix` + +**Description:** This flake defines a standalone environment for working with `x.py`, which appears to be a custom build system for Rust projects. It provides a development shell with necessary tools and a package that executes `test_json_output.py` to generate and validate JSON output, likely related to the `x.py` build process. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustSrcFlake`: `github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2` + * The same `rust` source flake used in the root `flake.nix`, providing the `src/stage0` path. +* `rustOverlay`: `github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify` + * The same `rust-overlay` used in the root `flake.nix`. + +**Outputs:** + +* **`devShells.aarch64-linux.default`:** + * A development shell named `standalonex-dev-shell`. + * **Packages Included:** `pkgs.python3`. + * **`shellHook`:** + * Adds the flake's source directory (`${self}/`) to `PATH`, making `x.py` directly executable. + * Sets `RUST_SRC_STAGE0_PATH` to the `src/stage0` directory from `rustSrcFlake`. + * Creates a `config.toml` file with paths to `rustc` and `cargo` from `pkgs.rust-bin.stable.latest.default`. + * Sets `RUST_BOOTSTRAP_CONFIG` to the path of the generated `config.toml`. + * Creates dummy `etc/` files (`rust_analyzer_settings.json`, `rust_analyzer_eglot.el`, `rust_analyzer_helix.toml`) which are likely expected by `x.py` or related tools. + +* **`packages.aarch64-linux.default`:** + * A Nix package named `xpy-build-output`. + * **`src`:** Uses the flake's own source (`self`) as input. + * **`nativeBuildInputs`:** `pkgs.python3` and `pkgs.jq`. + * **`phases`:** Explicitly defines `buildPhase` and `installPhase`. + * **`buildPhase`:** This is the most complex part: + * It creates a writable temporary directory (`$TMPDIR/xpy_work`) and copies the flake's source into it. + * It then copies `config.old.toml` to `config.toml` and uses `sed` to inject the correct `rustc` and `cargo` paths into `config.toml`. + * Sets `RUST_BOOTSTRAP_CONFIG` to the path of the modified `config.toml`. + * Sets `HOME` and `CARGO_HOME` to writable temporary directories. + * Executes `python3 test_json_output.py --output-dir $out` to generate JSON files. + * Validates the generated JSON files using `jq`. + * **`installPhase`:** Is empty, as the output is generated directly in the `buildPhase`. + +**Overall Purpose:** This flake is a self-contained environment for testing and generating output from the `x.py` build system. It meticulously sets up the necessary environment variables, configuration files, and dependencies to run `test_json_output.py`, which in turn uses `x.py` to produce JSON output. This output is then validated and exposed as a Nix package. This flake is crucial for understanding how the `x.py` build system is exercised and how its metadata is captured. + +## Standalone x.py Environment + +# Standalone x.py Environment + +This directory contains a standalone version of the `x.py` script from the Rust compiler build system. +It is packaged as a Nix flake that can be built and tested independently. + +## JSON Output Generation + +The flake provides a package that builds the Rust compiler in a "dry run" mode. +In this mode, the build commands are not actually executed, but are captured in JSON files. +This is useful for analyzing the build process and for creating alternative build systems. + +To build the package and generate the JSON files, run the following command from this directory: + +```bash +nix build +``` + +The generated JSON files will be in the `result` directory. + +### Sample JSON Output + +Here is a sample of one of the generated JSON files: + +```json +{ + "command": "/nix/store/lrr9mf5sg6qbas19z1ixjna024zkqws4-rust-default-1.90.0/bin/cargo", + "args": [ + "build", + "--manifest-path", + "/nix/store/qsclyr4nsd25i5p9al261blrki1l9w31-source/standalonex/src/bootstrap/Cargo.toml" + ], + "env": { + "SHELL": "/nix/store/hxmi7d6vbdgbzklm4icfk2y83ncw8la9-bash-5.3p3/bin/bash", + "RUST_BOOTSTRAP_JSON_OUTPUT_DIR": "/nix/store/sc437kd47w1bajlcrdmmgdg0ng57f1l5-xpy-build-output-0.1.0", + "..." + }, + "cwd": "/nix/store/qsclyr4nsd25i5p9al261blrki1l9w31-source/standalonex", + "type": "rust_compiler_invocation" +} +``` + +### Field Explanations + +- `command`: The command to be executed. +- `args`: A list of arguments for the command. +- `env`: A dictionary of environment variables for the command. +- `cwd`: The working directory in which the command should be executed. +- `type`: The type of the invocation. In this case, it's a rust compiler invocation. + +## Bootstrap Builder Flake + +# Bootstrap Builder Flake + +This flake is responsible for building the Rust bootstrap compiler from source. + +## Plan: +1. Create a `flake.nix` file in this directory that builds the `bootstrap` compiler from the rust source. +2. The `rust-src` will be an input to this flake, using a github URL with a specific git hash. +3. The build will use `pkgs.rustPlatform.buildRustPackage`. +4. After the `bootstrap` compiler is built, it will be used by the `standalonex` flake to generate the JSON output of the full Rust build process. +5. The findings will then be documented in the `README.md` of the `standalonex` directory. + +## build_helper + +Types and functions shared across tools in this workspace. + +--- +**Note:** This `README.md` is a consolidation of several documentation files for easier access. The original files were: +- `CONFIGURATION.md` +- `NIX_FLAKES_DOCUMENTATION.md` +- `OVERVIEW.md` +- `standalonex/README.md` +- `flakes/bootstrap-builder/README.md` +- `standalonex/src/build_helper/README.md` \ No newline at end of file diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..8dfbd5ef --- /dev/null +++ b/TODO.md @@ -0,0 +1,56 @@ +# TODO List for rust-bootstrap-nix Project + +This document outlines the immediate next steps and ongoing tasks for the `rust-bootstrap-nix` project. + +## Work Done (Summary of recent progress): + +* **Rust Precondition Checks:** Converted the logic from `run_preconditions_test.sh` and `test_nix_preconditions.sh` into Rust, implemented in `bootstrap-config-builder/src/preconditions.rs`. +* **`bootstrap-config-builder` Refactoring:** The `bootstrap-config-builder/src/utils.rs` module has been refactored into a more organized structure with sub-modules. +* **Logging & Dry-Run:** Added comprehensive logging and a `--dry-run` option to the `bootstrap-config-builder` for better visibility and testing. +* **`nix-dir` Tool:** Created a new binary tool (`nix-dir`) to inspect Nix flakes and their attributes. +* **Configuration System Refactoring (`bootstrap-config-builder`):** + * Implemented `config.toml` loading and merging with command-line overrides. + * Resolved build errors related to `toml`, `serde` dependencies, and argument parsing. +* **Rust Bootstrap Configuration Consumption (`standalonex/src/bootstrap`):** + * Created `local_nix_config.rs` for `LocalNixConfig`. + * Declared `local_nix_config` and `nix_config` modules. + * Added `nix: Option` field to `LocalTomlConfig`. + * Created `nix_config.rs` with `NixConfigApplicator`. + * Integrated `NixConfigApplicator` into `parse.rs`. + * Added Nix-related fields to `ParsedConfig`. +* **Documentation:** + * Documented the Nix-Rust integration process in `docs/Nix_Rust_Integration_Process.md`. +* **`lib.rs` Refactoring (`standalonex/src/bootstrap/src/lib.rs`):** + * Started refactoring `lib.rs` into smaller, more manageable files. + * Created `compiler.rs`, `enums.rs`, `constants.rs`, `crate_struct.rs`, `dependency_type.rs`, `helpers.rs`, `build_struct.rs`, `build_impl_new.rs`, `build_impl_main.rs`, `build_impl_paths.rs`, `build_impl_tools.rs`, `build_impl_utils.rs`, `build_impl_submodules.rs`, `build_impl_config.rs`. + +## Next Steps: + +### 1. Complete `lib.rs` Refactoring + +* Finish moving all content from `standalonex/src/bootstrap/src/lib.rs` to the newly created files. +* Update `standalonex/src/bootstrap/src/lib.rs` to correctly declare and re-export all modules. +* Ensure the project compiles successfully after refactoring. + +### 2. Integrate Nix Interaction into Rust Bootstrap + +* **Read Nix Configuration:** Utilize the Nix-related paths and flake references from `ParsedConfig` within the Rust bootstrap logic. +* **Interact with Nix:** Implement logic to execute Nix commands (e.g., `nix eval`, `nix build`) from within Rust to resolve actual Nix store paths for `rustSrcFlake` and other inputs. +* **Integrate Resolved Paths:** Incorporate these resolved Nix store paths into the Rust build process to build the specified version of Rust through the 8-level bootstrap. + +### 3. Refine `nix-dir` Tool + +* **Detailed Output:** Enhance the `nix-dir` tool to provide more detailed output for flake attributes, including types and descriptions. +* **Filtering & Searching:** Implement capabilities for filtering and searching flake attributes. +* **JSON Output:** Add a `--json` output option for programmatic use and easier integration with other tools. + +### 4. Integrate `bootstrap-config-builder` into the Build Process + +* **Makefile Integration:** Create a robust Makefile target to run `bootstrap-config-builder` to generate `config.toml` as a prerequisite for the main build process. +* **`config.toml` Consumption:** Ensure the generated `config.toml` is correctly consumed and utilized by the Rust bootstrap process. + +### 5. Continue with Overall Project Goals + +* **Define Packages/Applications:** Further define and refine packages and applications within the Nix flake. +* **Build & Test Commands:** Set up comprehensive build and test commands for the entire project. +* **Refine `devShell`:** Continue to refine the `devShell` environment for optimal development experience. \ No newline at end of file diff --git a/bootstrap-config-builder/Cargo.toml b/bootstrap-config-builder/Cargo.toml new file mode 100644 index 00000000..b0e49823 --- /dev/null +++ b/bootstrap-config-builder/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "bootstrap-config-builder" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5.4", features = ["derive"] } +anyhow = "1.0.86" +serde_json = "1.0.117" +toml = "0.8.12" +serde = { version = "1.0.198", features = ["derive"] } +log = "0.4.21" +env_logger = "0.11.3" +git2 = "0.18.3" +toml_edit = "0.22.14" + +[dev-dependencies] +tempfile = "3.10.1" + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "nix-dir" +path = "src/bin/nix-dir.rs" + +[[bin]] +name = "bootstrap-config-generator" +path = "src/bin/bootstrap-config-generator.rs" \ No newline at end of file diff --git a/bootstrap-config-builder/flake.lock b/bootstrap-config-builder/flake.lock new file mode 100644 index 00000000..7c236a5d --- /dev/null +++ b/bootstrap-config-builder/flake.lock @@ -0,0 +1,170 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rustOverlay": "rustOverlay", + "rustSrcFlake": "rustSrcFlake" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_4" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustOverlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustSrcFlake": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_3", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "3487cd3843083db70ee30023f19344568ade9c9f", + "repo": "rust", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/bootstrap-config-builder/flake.nix b/bootstrap-config-builder/flake.nix new file mode 100644 index 00000000..06c517cc --- /dev/null +++ b/bootstrap-config-builder/flake.nix @@ -0,0 +1,42 @@ +{ + description = "Flake to generate config.toml for rust-bootstrap-nix"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rustSrcFlake.url = "github:meta-introspector/rust?ref=3487cd3843083db70ee30023f19344568ade9c9f"; + rustOverlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, rustSrcFlake, rustOverlay }: + let + pkgs = import nixpkgs { + system = "aarch64-linux"; + overlays = [ rustOverlay.overlays.default ]; + }; + rustcPath = "${pkgs.rust-bin.stable."1.89.0".default}/bin/rustc"; # Using a specific rustc version + cargoPath = "${pkgs.cargo}/bin/cargo"; # Using cargo from nixpkgs + projectRoot = "/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix"; # Absolute path to the main project root + rustSrcFlakePath = "/data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src"; # Absolute path to rust-src + in + { + packages.aarch64-linux.generatedConfigToml = pkgs.runCommand "generated-config.toml" + { + nativeBuildInputs = [ pkgs.cargo pkgs.rustc ]; # Ensure cargo and rustc are available + } '' + # Build the bootstrap-config-generator binary + ${pkgs.cargo}/bin/cargo build --release --bin bootstrap-config-generator --target aarch64-unknown-linux-gnu + + # Run the generator to create config.toml + ./target/release/bootstrap-config-generator \ + --rustc-path ${rustcPath} \ + --cargo-path ${cargoPath} \ + --project-root ${projectRoot} \ + --rust-src-flake-path ${rustSrcFlakePath} \ + --output $out/config.toml + ''; + }; +} + + + + diff --git a/bootstrap-config-builder/generated_config.toml b/bootstrap-config-builder/generated_config.toml new file mode 100644 index 00000000..e45c4db3 --- /dev/null +++ b/bootstrap-config-builder/generated_config.toml @@ -0,0 +1,56 @@ +# Generated by bootstrap-config-builder +# +# System: +# Project Root: /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix + +[nix] +nixpkgs_path = "" +base_branch = "feature/CRQ-016-nixify" +rust_overlay_path = "" +rust_bootstrap_nix_path = "" +configuration_nix_path = "" +rust_src_flake_path = "/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src" +rust_bootstrap_nix_flake_ref = "" +rust_src_flake_ref = "" +date = "2025-01-30" + +[rust] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.84.1/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.84.1/bin/cargo" +channel = "stable" +download-rustc = false +parallel-compiler = false +llvm-tools = false +debuginfo-level = 0 + +[build] +stage = "" +target = "" +patch-binaries-for-nix = false +vendor = false +build-dir = "" +jobs = 0 + +[env] +HOME = "" +CARGO_HOME = "" + +[install] +prefix = "" +sysconfdir = "" + +[dist] +sign-folder = "" +upload-addr = "" + +[llvm] +download-ci-llvm = false +ninja = false + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "" diff --git a/bootstrap-config-builder/src/args.rs b/bootstrap-config-builder/src/args.rs new file mode 100644 index 00000000..af7db985 --- /dev/null +++ b/bootstrap-config-builder/src/args.rs @@ -0,0 +1,143 @@ +use clap::Parser; +use std::path::PathBuf; + +/// A tool to generate config.toml for the rust-bootstrap process by querying Nix flakes. +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + /// Path to a config.toml file to load configuration from. + #[arg(long, short = 'c', value_name = "FILE")] + pub config_file: Option, + + /// The build stage to configure (e.g., "0", "1", "2"). + #[arg(long, short, value_name = "STAGE")] + pub stage: Option, + + /// The target triple for the build (e.g., "aarch64-unknown-linux-gnu"). + #[arg(long, short, value_name = "TARGET")] + pub target: Option, + + /// The root directory of the rust-bootstrap-nix project. + #[arg(long, value_name = "PATH")] + pub project_root: Option, + + /// The system for which to build (e.g., "aarch64-linux"). + #[arg(long, value_name = "SYSTEM")] + pub system: Option, + + /// The output path for the generated config.toml. + #[arg(long, short, default_value = "config.toml")] + pub output: Option, + + /// The flake reference for rust-bootstrap-nix. + #[arg(long, value_name = "REF")] + pub rust_bootstrap_nix_flake_ref: Option, + + /// The flake reference for rust-src. + #[arg(long, value_name = "REF")] + pub rust_src_flake_ref: Option, + + /// The path to nixpkgs. + #[arg(long, value_name = "PATH")] + pub nixpkgs_path: Option, + + /// The path to rust-overlay. + #[arg(long, value_name = "PATH")] + pub rust_overlay_path: Option, + + /// The path to rust-bootstrap-nix. + #[arg(long, value_name = "PATH")] + pub rust_bootstrap_nix_path: Option, + + /// The path to configuration-nix. + #[arg(long, value_name = "PATH")] + pub configuration_nix_path: Option, + + /// The path to rust-src flake. + #[arg(long, value_name = "PATH")] + pub rust_src_flake_path: Option, + + /// Perform a dry run, printing the generated config to stdout. + #[arg(long)] + pub dry_run: bool, + + /// The path to the rustc executable. + #[arg(long, value_name = "PATH")] + pub rustc_path: Option, + + /// The path to the cargo executable. + #[arg(long, value_name = "PATH")] + pub cargo_path: Option, + + /// The Rust channel to use (e.g., "stable", "beta", "nightly"). + #[arg(long, value_name = "CHANNEL")] + pub rust_channel: Option, + + /// Whether to download rustc. + #[arg(long)] + pub rust_download_rustc: Option, + + /// Whether to enable parallel compilation. + #[arg(long)] + pub rust_parallel_compiler: Option, + + /// Whether to enable LLVM tools. + #[arg(long)] + pub rust_llvm_tools: Option, + + /// The debuginfo level for Rust compilation. + #[arg(long, value_name = "LEVEL")] + pub rust_debuginfo_level: Option, + + /// Whether to patch binaries for Nix. + #[arg(long)] + pub patch_binaries_for_nix: Option, + + /// Whether to enable vendoring. + #[arg(long)] + pub vendor: Option, + + /// The build directory. + #[arg(long, value_name = "PATH")] + pub build_dir: Option, + + /// The number of build jobs. + #[arg(long, value_name = "JOBS")] + pub build_jobs: Option, + + /// The HOME directory for the build. + #[arg(long, value_name = "PATH")] + pub home_dir: Option, + + /// The CARGO_HOME directory for the build. + #[arg(long, value_name = "PATH")] + pub cargo_home_dir: Option, + + /// The installation prefix. + #[arg(long, value_name = "PATH")] + pub install_prefix: Option, + + /// The system configuration directory. + #[arg(long, value_name = "PATH")] + pub install_sysconfdir: Option, + + /// The folder for distribution signing. + #[arg(long, value_name = "PATH")] + pub dist_sign_folder: Option, + + /// The upload address for distribution. + #[arg(long, value_name = "ADDR")] + pub dist_upload_addr: Option, + + /// Whether to download CI LLVM. + #[arg(long)] + pub llvm_download_ci_llvm: Option, + + /// Whether to use Ninja for LLVM. + #[arg(long)] + pub llvm_ninja: Option, + + /// The change ID for tracking major changes. + #[arg(long, value_name = "ID")] + pub change_id: Option, +} diff --git a/bootstrap-config-builder/src/bin/bootstrap-config-generator.rs b/bootstrap-config-builder/src/bin/bootstrap-config-generator.rs new file mode 100644 index 00000000..424d2fbf --- /dev/null +++ b/bootstrap-config-builder/src/bin/bootstrap-config-generator.rs @@ -0,0 +1,109 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use std::fs; +use log::{info, debug}; // Import log macros +use toml; +use bootstrap_config_builder::config::AppConfig; + +use bootstrap_config_builder::preconditions; +use bootstrap_config_builder::utils::validate_project_root::validate_project_root; +use bootstrap_config_builder::utils::construct_config_content::construct_config_content; +use bootstrap_config_builder::args::Args; + +fn main() -> Result<()> { + env_logger::init(); // Initialize the logger + + let args = Args::parse(); + debug!("Raw CLI Arguments: {:?}", args); + + let mut app_config = if let Some(config_file_path) = &args.config_file { + info!("Loading configuration from file: {:?}", config_file_path); + let config_content = fs::read_to_string(config_file_path) + .context(format!("Failed to read config file: {:?}", config_file_path))?; + toml::from_str(&config_content) + .context(format!("Failed to parse config file: {:?}", config_file_path))? + } else { + AppConfig::default() + }; + + app_config.merge_with_args(&args); + info!("Final merged configuration: {:?}", app_config); + + info!("Starting config generation for stage {:?} and target {:?}", app_config.stage, app_config.target); + + // Run precondition checks + info!("Running precondition checks..."); + preconditions::check_nix_command_available()?; + info!("Nix command available."); + + // 1. Validate the project root + info!("Validating project root: {:?}", app_config.project_root); + let project_root = validate_project_root(app_config.project_root.as_ref().context("Project root is required")?)?; + let flake_path_str = project_root.to_str() + .context("Project root path contains non-UTF8 characters")?; + info!("Project root validated: {}", flake_path_str); + + // 2. Use provided flake input paths + let rust_src_flake_path_lossy = app_config.rust_src_flake_path + .as_ref() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_default(); + + debug!("rust_src_flake_path: {:?}", rust_src_flake_path_lossy); + + preconditions::check_rust_toolchain_sysroot( + &rust_src_flake_path_lossy, + )?; + info!("Rust toolchain sysroot check passed."); + + // 3. Construct the config.toml content + info!("Constructing config.toml content..."); + let config_content = construct_config_content( + app_config.system.as_deref().unwrap_or_default(), + flake_path_str, + app_config.nixpkgs_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.rust_overlay_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.rust_bootstrap_nix_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.configuration_nix_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.rust_src_flake_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.stage.as_deref().unwrap_or_default(), + app_config.target.as_deref().unwrap_or_default(), + app_config.rust_bootstrap_nix_flake_ref.as_deref().unwrap_or_default(), + app_config.rust_src_flake_ref.as_deref().unwrap_or_default(), + app_config.rustc_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.cargo_path.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.rust_channel.as_deref().unwrap_or("stable"), + app_config.rust_download_rustc.unwrap_or(false), + app_config.rust_parallel_compiler.unwrap_or(false), + app_config.rust_llvm_tools.unwrap_or(false), + app_config.rust_debuginfo_level.unwrap_or(0), + app_config.patch_binaries_for_nix.unwrap_or(false), + app_config.vendor.unwrap_or(false), + app_config.build_dir.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.build_jobs.unwrap_or(0), + app_config.home_dir.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.cargo_home_dir.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.install_prefix.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.install_sysconfdir.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.dist_sign_folder.as_deref().map(|p| p.to_str().unwrap_or_default()).unwrap_or_default(), + app_config.dist_upload_addr.as_deref().unwrap_or_default(), + app_config.llvm_download_ci_llvm.unwrap_or(false), + app_config.llvm_ninja.unwrap_or(false), + app_config.change_id.as_deref().unwrap_or_default(), + ); + debug!("Generated config content:\n{}", config_content); + + // 4. Handle output based on dry_run flag + if app_config.dry_run.unwrap_or(false) { + info!("Dry run enabled. Generated config will be printed to stdout."); + println!("{}", config_content); + } else { + let output_path = app_config.output.unwrap_or_else(|| "config.toml".into()); + info!("Writing generated config to file: {:?}", output_path); + fs::write(&output_path, config_content) + .context(format!("Failed to write config to file: {:?}", output_path))?; + info!("Config successfully written to {:?}", output_path); + } + + Ok(()) +} diff --git a/bootstrap-config-builder/src/bin/nix-dir.rs b/bootstrap-config-builder/src/bin/nix-dir.rs new file mode 100644 index 00000000..0be58a96 --- /dev/null +++ b/bootstrap-config-builder/src/bin/nix-dir.rs @@ -0,0 +1,79 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use std::process::Command; +use serde_json::Value; +use log::{info, debug}; + +/// A tool to inspect Nix flakes and their attributes. +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + /// The flake reference to inspect (e.g., "nixpkgs", "github:NixOS/nixpkgs/nixos-23.11") + #[arg()] + flake_ref: String, + + /// Output raw JSON from 'nix flake show' command. + #[arg(long, default_value_t = false)] + json: bool, +} + +fn main() -> Result<()> { + env_logger::init(); // Initialize the logger + + let args = Args::parse(); + + info!("Inspecting Nix flake: {}", args.flake_ref); + + let mut command = Command::new("nix"); + command.args(&["flake", "show", "--json", &args.flake_ref]); + + debug!("Executing Nix command: {:?}", command); + + let output = command.output() + .with_context(|| format!("Failed to execute nix flake show for '{}'", args.flake_ref))?; + + if !output.status.success() { + anyhow::bail!( + "Nix command failed for flake show '{}':\n{}\nStderr: {}", + args.flake_ref, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + let json_output: Value = serde_json::from_slice(&output.stdout) + .with_context(|| "Failed to parse nix flake show JSON output")?; + + if args.json { + println!("{}", serde_json::to_string_pretty(&json_output)?); + return Ok(()); + } + + println!("Flake Attributes for {}:", args.flake_ref); + + if let Some(inputs) = json_output.get("inputs") { + println!("\nInputs:"); + if let Some(inputs_obj) = inputs.as_object() { + for (key, _) in inputs_obj { + println!(" - {}", key); + } + } + } + + if let Some(outputs) = json_output.get("outputs") { + println!("\nOutputs:"); + if let Some(outputs_obj) = outputs.as_object() { + for (system, system_outputs) in outputs_obj { + println!(" {}:", system); + if let Some(system_outputs_obj) = system_outputs.as_object() { + for (key, value) in system_outputs_obj { + let output_type = value.get("type").and_then(|t| t.as_str()).unwrap_or("unknown"); + println!(" - {}: {}", key, output_type); + } + } + } + } + } + + Ok(()) +} diff --git a/bootstrap-config-builder/src/compose_path.rs b/bootstrap-config-builder/src/compose_path.rs new file mode 100644 index 00000000..5dbf299c --- /dev/null +++ b/bootstrap-config-builder/src/compose_path.rs @@ -0,0 +1 @@ +//compose_path diff --git a/bootstrap-config-builder/src/config.rs b/bootstrap-config-builder/src/config.rs new file mode 100644 index 00000000..ed7bab39 --- /dev/null +++ b/bootstrap-config-builder/src/config.rs @@ -0,0 +1,77 @@ +use serde::Deserialize; +use std::path::PathBuf; + +#[derive(Debug, Default, Deserialize)] +pub struct AppConfig { + pub stage: Option, + pub target: Option, + pub project_root: Option, + pub system: Option, + pub output: Option, + pub rust_bootstrap_nix_flake_ref: Option, + pub rust_src_flake_ref: Option, + pub nixpkgs_path: Option, + pub rust_overlay_path: Option, + pub rust_bootstrap_nix_path: Option, + pub configuration_nix_path: Option, + pub rust_src_flake_path: Option, + pub dry_run: Option, + pub rustc_path: Option, + pub cargo_path: Option, + pub rust_channel: Option, + pub rust_download_rustc: Option, + pub rust_parallel_compiler: Option, + pub rust_llvm_tools: Option, + pub rust_debuginfo_level: Option, + pub patch_binaries_for_nix: Option, + pub vendor: Option, + pub build_dir: Option, + pub build_jobs: Option, + pub home_dir: Option, + pub cargo_home_dir: Option, + pub install_prefix: Option, + pub install_sysconfdir: Option, + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub llvm_download_ci_llvm: Option, + pub llvm_ninja: Option, + pub change_id: Option, +} + +impl AppConfig { + pub fn merge_with_args(&mut self, args: &crate::args::Args) { + if let Some(stage) = args.stage.clone() { self.stage = Some(stage); } + if let Some(target) = args.target.clone() { self.target = Some(target); } + if let Some(project_root) = args.project_root.clone() { self.project_root = Some(project_root); } + if let Some(system) = args.system.clone() { self.system = Some(system); } + if let Some(output) = args.output.clone() { self.output = Some(output); } + if let Some(rust_bootstrap_nix_flake_ref) = args.rust_bootstrap_nix_flake_ref.clone() { self.rust_bootstrap_nix_flake_ref = Some(rust_bootstrap_nix_flake_ref); } + if let Some(rust_src_flake_ref) = args.rust_src_flake_ref.clone() { self.rust_src_flake_ref = Some(rust_src_flake_ref); } + if let Some(nixpkgs_path) = args.nixpkgs_path.clone() { self.nixpkgs_path = Some(nixpkgs_path); } + if let Some(rust_overlay_path) = args.rust_overlay_path.clone() { self.rust_overlay_path = Some(rust_overlay_path); } + if let Some(rust_bootstrap_nix_path) = args.rust_bootstrap_nix_path.clone() { self.rust_bootstrap_nix_path = Some(rust_bootstrap_nix_path); } + if let Some(configuration_nix_path) = args.configuration_nix_path.clone() { self.configuration_nix_path = Some(configuration_nix_path); } + if let Some(rust_src_flake_path) = args.rust_src_flake_path.clone() { self.rust_src_flake_path = Some(rust_src_flake_path); } + self.dry_run = Some(args.dry_run); + if let Some(rustc_path) = args.rustc_path.clone() { self.rustc_path = Some(rustc_path); } + if let Some(cargo_path) = args.cargo_path.clone() { self.cargo_path = Some(cargo_path); } + if let Some(rust_channel) = args.rust_channel.clone() { self.rust_channel = Some(rust_channel); } + if let Some(rust_download_rustc) = args.rust_download_rustc { self.rust_download_rustc = Some(rust_download_rustc); } + if let Some(rust_parallel_compiler) = args.rust_parallel_compiler { self.rust_parallel_compiler = Some(rust_parallel_compiler); } + if let Some(rust_llvm_tools) = args.rust_llvm_tools { self.rust_llvm_tools = Some(rust_llvm_tools); } + if let Some(rust_debuginfo_level) = args.rust_debuginfo_level { self.rust_debuginfo_level = Some(rust_debuginfo_level); } + if let Some(patch_binaries_for_nix) = args.patch_binaries_for_nix { self.patch_binaries_for_nix = Some(patch_binaries_for_nix); } + if let Some(vendor) = args.vendor { self.vendor = Some(vendor); } + if let Some(build_dir) = args.build_dir.clone() { self.build_dir = Some(build_dir); } + if let Some(build_jobs) = args.build_jobs { self.build_jobs = Some(build_jobs); } + if let Some(home_dir) = args.home_dir.clone() { self.home_dir = Some(home_dir); } + if let Some(cargo_home_dir) = args.cargo_home_dir.clone() { self.cargo_home_dir = Some(cargo_home_dir); } + if let Some(install_prefix) = args.install_prefix.clone() { self.install_prefix = Some(install_prefix); } + if let Some(install_sysconfdir) = args.install_sysconfdir.clone() { self.install_sysconfdir = Some(install_sysconfdir); } + if let Some(dist_sign_folder) = args.dist_sign_folder.clone() { self.dist_sign_folder = Some(dist_sign_folder); } + if let Some(dist_upload_addr) = args.dist_upload_addr.clone() { self.dist_upload_addr = Some(dist_upload_addr); } + if let Some(llvm_download_ci_llvm) = args.llvm_download_ci_llvm { self.llvm_download_ci_llvm = Some(llvm_download_ci_llvm); } + if let Some(llvm_ninja) = args.llvm_ninja { self.llvm_ninja = Some(llvm_ninja); } + if let Some(change_id) = args.change_id.clone() { self.change_id = Some(change_id); } + } +} diff --git a/bootstrap-config-builder/src/example.toml b/bootstrap-config-builder/src/example.toml new file mode 100644 index 00000000..bf1f3c8f --- /dev/null +++ b/bootstrap-config-builder/src/example.toml @@ -0,0 +1,54 @@ +# Generated by bootstrap-config-builder +# +# System: {system} +# Project Root: {flake_path_str} + +[nix] +nixpkgs_path = "{nixpkgs_path}" +rust_overlay_path = "{rust_overlay_path}" +rust_bootstrap_nix_path = "{rust_bootstrap_nix_path}" +configuration_nix_path = "{configuration_nix_path}" +rust_src_flake_path = "{rust_src_flake_path}" +rust_bootstrap_nix_flake_ref = "{rust_bootstrap_nix_flake_ref}" +rust_src_flake_ref = "{rust_src_flake_ref}" + +[rust] +rustc = "{rustc_path}" +cargo = "{cargo_path}" +channel = "{rust_channel}" +download-rustc = {rust_download_rustc} +parallel-compiler = {rust_parallel_compiler} +llvm-tools = {rust_llvm_tools} +debuginfo-level = {rust_debuginfo_level} + +[build] +stage = {stage} +target = "{target}" +patch-binaries-for-nix = {patch_binaries_for_nix} +vendor = {vendor} +build-dir = "{build_dir}" +jobs = {build_jobs} + +[env] +HOME = "{home_dir}" +CARGO_HOME = "{cargo_home_dir}" + +[install] +prefix = "{install_prefix}" +sysconfdir = "{install_sysconfdir}" + +[dist] +sign-folder = "{dist_sign_folder}" +upload-addr = "{dist_upload_addr}" + +[llvm] +download-ci-llvm = {llvm_download_ci_llvm} +ninja = {llvm_ninja} + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "{change_id}" diff --git a/bootstrap-config-builder/src/lib.rs b/bootstrap-config-builder/src/lib.rs new file mode 100644 index 00000000..582fd105 --- /dev/null +++ b/bootstrap-config-builder/src/lib.rs @@ -0,0 +1,17 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use std::fs; +use log::{info, debug}; // Import log macros +use toml; +use crate::config::AppConfig; + +pub mod utils; // Declare the utils module as public +pub mod preconditions; // Declare the preconditions module + +pub mod args; // Declare the args module +pub mod config; // Declare the config module + +use crate::utils::validate_project_root::validate_project_root; +use crate::utils::construct_config_content::construct_config_content; +use crate::args::Args; + diff --git a/bootstrap-config-builder/src/preconditions.rs b/bootstrap-config-builder/src/preconditions.rs new file mode 100644 index 00000000..d6d4999c --- /dev/null +++ b/bootstrap-config-builder/src/preconditions.rs @@ -0,0 +1,28 @@ +use anyhow::{Context, Result}; +use std::process::Command; +// use crate::utils::get_flake_input::get_flake_input; // Commented out + +pub fn check_nix_command_available() -> Result<()> { + Command::new("nix") + .arg("--version") + .output() + .with_context(|| "Failed to execute 'nix --version'. Is Nix installed and in PATH?")? + .status + .success() + .then_some(()) + .with_context(|| "'nix' command not found or failed to execute. Please install Nix.") +} + +pub fn check_rust_toolchain_sysroot( + rust_src_flake_path: &str, +) -> Result<()> { + // Simplified check: just verify if the rust_src_flake_path exists and contains src/ci/channel + let known_file = format!("{}/src/ci/channel", rust_src_flake_path); + + if std::path::Path::new(&known_file).exists() { + Ok(()) + } else { + anyhow::bail!("Rust source flake NOT found at: {}. Known file 'src/ci/channel' missing.", rust_src_flake_path); + } +} + diff --git a/bootstrap-config-builder/src/utils/compose_path.rs b/bootstrap-config-builder/src/utils/compose_path.rs new file mode 100644 index 00000000..13be56ce --- /dev/null +++ b/bootstrap-config-builder/src/utils/compose_path.rs @@ -0,0 +1,3 @@ +pub fn compose_path(path_expr: &str, path_template: &str) -> String { + path_expr.replacen("{}", &format!("{}", path_template), 1) +} \ No newline at end of file diff --git a/bootstrap-config-builder/src/utils/construct_config_content.rs b/bootstrap-config-builder/src/utils/construct_config_content.rs new file mode 100644 index 00000000..69eaa7e9 --- /dev/null +++ b/bootstrap-config-builder/src/utils/construct_config_content.rs @@ -0,0 +1,70 @@ +use super::format_file; // Import from sibling module + +#[allow(clippy::too_many_arguments)] +pub fn construct_config_content( + system: &str, + flake_path_str: &str, + nixpkgs_path: &str, + rust_overlay_path: &str, + rust_bootstrap_nix_path: &str, + configuration_nix_path: &str, + rust_src_flake_path: &str, + stage: &str, + target: &str, + rust_bootstrap_nix_flake_ref: &str, + rust_src_flake_ref: &str, + rustc_path: &str, + cargo_path: &str, + rust_channel: &str, + rust_download_rustc: bool, + rust_parallel_compiler: bool, + rust_llvm_tools: bool, + rust_debuginfo_level: u8, + patch_binaries_for_nix: bool, + vendor: bool, + build_dir: &str, + build_jobs: u32, + home_dir: &str, + cargo_home_dir: &str, + install_prefix: &str, + install_sysconfdir: &str, + dist_sign_folder: &str, + dist_upload_addr: &str, + llvm_download_ci_llvm: bool, + llvm_ninja: bool, + change_id: &str, +) -> String { + format_file::format_file( + system, + flake_path_str, + nixpkgs_path, + rust_overlay_path, + rust_bootstrap_nix_path, + configuration_nix_path, + rust_src_flake_path, + rust_bootstrap_nix_flake_ref, + rust_src_flake_ref, + stage, + target, + rustc_path, + cargo_path, + rust_channel, + rust_download_rustc, + rust_parallel_compiler, + rust_llvm_tools, + rust_debuginfo_level, + patch_binaries_for_nix, + vendor, + build_dir, + build_jobs, + home_dir, + cargo_home_dir, + install_prefix, + install_sysconfdir, + dist_sign_folder, + dist_upload_addr, + llvm_download_ci_llvm, + llvm_ninja, + change_id + ) +} \ No newline at end of file diff --git a/bootstrap-config-builder/src/utils/format_file.rs b/bootstrap-config-builder/src/utils/format_file.rs new file mode 100644 index 00000000..64b8d6c3 --- /dev/null +++ b/bootstrap-config-builder/src/utils/format_file.rs @@ -0,0 +1,127 @@ +// use anyhow::{Context, Result}; // Commented out +use std::fs; + +#[allow(clippy::too_many_arguments)] +pub fn format_file( + system: &str, + flake_path_str: &str, + nixpkgs_path: &str, + rust_overlay_path: &str, + rust_bootstrap_nix_path: &str, + configuration_nix_path: &str, + rust_src_flake_path: &str, + rust_bootstrap_nix_flake_ref: &str, + rust_src_flake_ref: &str, + stage: &str, + target: &str, + rustc_path: &str, + cargo_path: &str, + rust_channel: &str, + rust_download_rustc: bool, + rust_parallel_compiler: bool, + rust_llvm_tools: bool, + rust_debuginfo_level: u8, + patch_binaries_for_nix: bool, + vendor: bool, + build_dir: &str, + build_jobs: u32, + home_dir: &str, + cargo_home_dir: &str, + install_prefix: &str, + install_sysconfdir: &str, + dist_sign_folder: &str, + dist_upload_addr: &str, + llvm_download_ci_llvm: bool, + llvm_ninja: bool, + change_id: &str, +) -> String { + let template_content = r#"# Generated by bootstrap-config-builder +# +# System: {system} +# Project Root: {flake_path_str} + +[nix] +nixpkgs_path = "{nixpkgs_path}" +rust_overlay_path = "{rust_overlay_path}" +rust_bootstrap_nix_path = "{rust_bootstrap_nix_path}" +configuration_nix_path = "{configuration_nix_path}" +rust_src_flake_path = "{rust_src_flake_path}" +rust_bootstrap_nix_flake_ref = "{rust_bootstrap_nix_flake_ref}" +rust_src_flake_ref = "{rust_src_flake_ref}" + +[rust] +rustc = "{rustc_path}" +cargo = "{cargo_path}" +channel = "{rust_channel}" +download-rustc = {rust_download_rustc} +parallel-compiler = {rust_parallel_compiler} +llvm-tools = {rust_llvm_tools} +debuginfo-level = {rust_debuginfo_level} + +[build] +stage = "{stage}" +target = "{target}" +patch-binaries-for-nix = {patch_binaries_for_nix} +vendor = {vendor} +build-dir = "{build_dir}" +jobs = {build_jobs} + +[env] +HOME = "{home_dir}" +CARGO_HOME = "{cargo_home_dir}" + +[install] +prefix = "{install_prefix}" +sysconfdir = "{install_sysconfdir}" + +[dist] +sign-folder = "{dist_sign_folder}" +upload-addr = "{dist_upload_addr}" + +[llvm] +download-ci-llvm = {llvm_download_ci_llvm} +ninja = {llvm_ninja} + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "{change_id}" +"#; + + // Use string replacement for each placeholder + template_content + .replace("{system}", system) + .replace("{flake_path_str}", flake_path_str) + .replace("{nixpkgs_path}", nixpkgs_path) + .replace("{rust_overlay_path}", rust_overlay_path) + .replace("{rust_bootstrap_nix_path}", rust_bootstrap_nix_path) + .replace("{configuration_nix_path}", configuration_nix_path) + .replace("{rust_src_flake_path}", rust_src_flake_path) + .replace("{rust_bootstrap_nix_flake_ref}", rust_bootstrap_nix_flake_ref) + .replace("{rust_src_flake_ref}", rust_src_flake_ref) + .replace("{stage}", stage) + .replace("{target}", target) + .replace("{rustc_path}", rustc_path) + .replace("{cargo_path}", cargo_path) + .replace("{rust_channel}", rust_channel) + .replace("{rust_download_rustc}", &rust_download_rustc.to_string()) + .replace("{rust_parallel_compiler}", &rust_parallel_compiler.to_string()) + .replace("{rust_llvm_tools}", &rust_llvm_tools.to_string()) + .replace("{rust_debuginfo_level}", &rust_debuginfo_level.to_string()) + .replace("{patch_binaries_for_nix}", &patch_binaries_for_nix.to_string()) + .replace("{vendor}", &vendor.to_string()) + .replace("{build_dir}", build_dir) + .replace("{build_jobs}", &build_jobs.to_string()) + .replace("{home_dir}", home_dir) + .replace("{cargo_home_dir}", cargo_home_dir) + .replace("{install_prefix}", install_prefix) + .replace("{install_sysconfdir}", install_sysconfdir) + .replace("{dist_sign_folder}", dist_sign_folder) + .replace("{dist_upload_addr}", dist_upload_addr) + .replace("{llvm_download_ci_llvm}", &llvm_download_ci_llvm.to_string()) + .replace("{llvm_ninja}", &llvm_ninja.to_string()) + .replace("{change_id}", change_id) +} diff --git a/bootstrap-config-builder/src/utils/format_new.rs b/bootstrap-config-builder/src/utils/format_new.rs new file mode 100644 index 00000000..cd470224 --- /dev/null +++ b/bootstrap-config-builder/src/utils/format_new.rs @@ -0,0 +1,3 @@ +pub fn format_new(template: &str, arg1: &str, arg2: &str) -> String { + template.replacen("{}", arg1, 1).replace("{}", arg2) +} diff --git a/bootstrap-config-builder/src/utils/generate_step1_flake.rs b/bootstrap-config-builder/src/utils/generate_step1_flake.rs new file mode 100644 index 00000000..22445dc6 --- /dev/null +++ b/bootstrap-config-builder/src/utils/generate_step1_flake.rs @@ -0,0 +1,50 @@ +pub fn generate_step1_flake( + nixpkgs_ref: &str, + rust_overlay_ref: &str, + rust_src_flake_ref: &str, + rust_bootstrap_nix_ref: &str, + rust_src_flake_path: &str, +) -> String { + format!( + r#"{{ + description = "Step 1: Generate config.toml"; + + inputs = {{ + nixpkgs.url = "github:meta-introspector/nixpkgs?ref={}"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref={}"; + rustSrcFlake.url = "github:meta-introspector/rust?ref={}"; + rust-bootstrap-nix.url = "github:meta-introspector/rust-bootstrap-nix?ref={}"; + }}; + + outputs = {{ self, nixpkgs, rust-overlay, rustSrcFlake, rust-bootstrap-nix, ... }}@inputs: + let + system = "aarch64-linux"; + pkgs = import nixpkgs {{ + inherit system; + overlays = [ rust-overlay.overlays.default ]; + }}; + in + {{ + packages.aarch64-linux.default = pkgs.stdenv.mkDerivation {{ + name = "generate-config"; + src = rust-bootstrap-nix; + buildInputs = [ pkgs.cargo pkgs.rustc pkgs.cacert pkgs.nix ]; + buildPhase = ''' + export CARGO_HOME=$(mktemp -d) + cargo run --bin bootstrap-config-generator -- --project-root . --rust-src-flake-path {} --version 1.84.1 --target aarch64-unknown-linux-gnu --stage 0 + '''; + installPhase = ''' + mkdir -p $out + cp config.toml $out/config.toml + '''; + }}; + }}; +}} +"#, + nixpkgs_ref, + rust_overlay_ref, + rust_src_flake_ref, + rust_bootstrap_nix_ref, + rust_src_flake_path + ) +} diff --git a/bootstrap-config-builder/src/utils/get_flake_input.rs b/bootstrap-config-builder/src/utils/get_flake_input.rs new file mode 100644 index 00000000..3c4d6511 --- /dev/null +++ b/bootstrap-config-builder/src/utils/get_flake_input.rs @@ -0,0 +1,36 @@ +use anyhow::{Context, Result}; +use std::process::Command; +use super::compose_path; // Import from sibling module +use super::format_new; // Import from sibling module +use log::debug; // Import only debug macro + +pub fn get_flake_input(flake_path_str: &str, input_name: &str) -> Result { + let path_template = "path:{}"; + let path_expr = "(builtins.getFlake {}).inputs.{}.outPath"; + + let composed_path = compose_path::compose_path(path_expr, path_template); + let expr = format_new::format_new(&composed_path, flake_path_str, input_name); + + let mut command = Command::new("nix"); + command.args(&["eval", "--raw", "--extra-experimental-features", "nix-command flakes", "--expr", &expr]); + + debug!("Executing Nix command: {:?}", command); + debug!("Working directory: {:?} +", std::env::current_dir()); + + let output = command.output() + .with_context(|| format!("Failed to execute nix eval for input '{}'", input_name))?; + + if !output.status.success() { + anyhow::bail!( + "Nix command failed for input '{}':\n{}\nStderr: {}", + input_name, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + let stdout = String::from_utf8(output.stdout)?.trim().to_string(); + debug!("Nix command stdout: {}", stdout); + Ok(stdout) +} \ No newline at end of file diff --git a/bootstrap-config-builder/src/utils/git.rs b/bootstrap-config-builder/src/utils/git.rs new file mode 100644 index 00000000..5a91cfd7 --- /dev/null +++ b/bootstrap-config-builder/src/utils/git.rs @@ -0,0 +1,60 @@ +use anyhow::{Context, Result}; +use git2::{Repository, Signature, Oid, Commit}; +use log::info; + +pub fn create_orphan_branch(repo_path: &str, branch_name: &str) -> Result<()> { + let repo = Repository::open(repo_path)?; + info!("Creating orphan branch '{}' in repo at '{}'", branch_name, repo_path); + + let head = repo.head()?; + let head_commit = head.peel_to_commit()?; + + let signature = Signature::now("bootstrap-config-generator", "bootstrap-config-generator@example.com")?; + + // Create an empty tree + let tree_id = Oid::from_str("4b825dc642cb6eb9a060e54bf8d69288fbee4904")?; + let tree = repo.find_tree(tree_id)?; + + // Create the commit + let commit_id = repo.commit( + Some("HEAD"), + &signature, + &signature, + "Initial commit for orphan branch", + &tree, + &[&head_commit], + )?; + + // Create the branch + repo.branch(branch_name, &repo.find_commit(commit_id)?, true)?; + + Ok(()) +} + +pub fn commit_files(repo_path: &str, files: &[&str], message: &str) -> Result<()> { + let repo = Repository::open(repo_path)?; + let mut index = repo.index()?; + + for file in files { + index.add_path(std::path::Path::new(file))?; + } + + index.write()?; + + let oid = index.write_tree()?; + let tree = repo.find_tree(oid)?; + + let signature = Signature::now("bootstrap-config-generator", "bootstrap-config-generator@example.com")?; + let parent_commit = repo.head()?.peel_to_commit()?; + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &[&parent_commit], + )?; + + Ok(()) +} diff --git a/bootstrap-config-builder/src/utils/git_command_helpers.rs b/bootstrap-config-builder/src/utils/git_command_helpers.rs new file mode 100644 index 00000000..dd4ad936 --- /dev/null +++ b/bootstrap-config-builder/src/utils/git_command_helpers.rs @@ -0,0 +1,55 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use anyhow::{Context, Result}; + +/// Runs a git command and returns the output, handling errors. +pub fn run_git_command( + current_dir: &Path, + args: &[&str], + error_message: &str, + dry_run: bool, +) -> Result<()> { + println!("Running git command in CWD: {:?}", current_dir); + let command_str = format!("git {}", args.join(" ")); + if dry_run { + println!("Dry run: Would execute: {}", command_str); + return Ok(()) + } + println!("Executing: {}", command_str); + + let output = Command::new("git") + .current_dir(current_dir) + .args(args) + .output() + .with_context(|| format!("Failed to execute git command: {}", command_str))?; + + if !output.status.success() { + anyhow::bail!( + "Git command failed: {} +Stdout: {} +Stderr: {}", + error_message, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +/// Runs a command and returns the stdout as a String. +pub fn output_result(cmd: &mut Command) -> Result { + let output = cmd.stderr(Stdio::inherit()).output() + .with_context(|| format!("Failed to run command: {:?}", cmd))?; + + if !output.status.success() { + anyhow::bail!( + "Command did not execute successfully: {:?}\nExpected success, got: {} +Stderr: {}", + cmd, + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8(output.stdout) + .with_context(|| "Failed to convert stdout to UTF-8".to_string()) +} diff --git a/bootstrap-config-builder/src/utils/mod.rs b/bootstrap-config-builder/src/utils/mod.rs new file mode 100644 index 00000000..8626731f --- /dev/null +++ b/bootstrap-config-builder/src/utils/mod.rs @@ -0,0 +1,9 @@ +pub mod compose_path; +pub mod construct_config_content; +pub mod format_file; +pub mod format_new; +pub mod get_flake_input; +pub mod validate_project_root; +pub mod git; +pub mod generate_step1_flake; +pub mod git_command_helpers; diff --git a/bootstrap-config-builder/src/utils/validate_project_root.rs b/bootstrap-config-builder/src/utils/validate_project_root.rs new file mode 100644 index 00000000..e8bafb60 --- /dev/null +++ b/bootstrap-config-builder/src/utils/validate_project_root.rs @@ -0,0 +1,15 @@ +use anyhow::{Context, Result}; +use std::{ + fs, + path::PathBuf, +}; + +pub fn validate_project_root(project_root: &PathBuf) -> Result { + let canonicalized_root = fs::canonicalize(project_root) + .with_context(|| format!("Failed to find absolute path for project root: {:?}", project_root))?; + + if !canonicalized_root.join("flake.nix").exists() { + anyhow::bail!("flake.nix not found in the specified project root: {:?}", canonicalized_root); + } + Ok(canonicalized_root) +} \ No newline at end of file diff --git a/bootstrap-config-builder/tests/config_integration_test.rs b/bootstrap-config-builder/tests/config_integration_test.rs new file mode 100644 index 00000000..8fdf170b --- /dev/null +++ b/bootstrap-config-builder/tests/config_integration_test.rs @@ -0,0 +1,230 @@ +use bootstrap_config_builder::config::AppConfig; +use bootstrap_config_builder::args::Args; +use bootstrap_config_builder::utils::format_file; +use std::path::PathBuf; + +#[test] +fn test_app_config_deserialization() { + let toml_content = r#"## + stage = "test_stage" + target = "test_target" + project_root = "/test/project_root" + system = "test_system" + output = "/test/output" + rust_bootstrap_nix_flake_ref = "test_bootstrap_ref" + rust_src_flake_ref = "test_src_ref" + nixpkgs_path = "/test/nixpkgs" + rust_overlay_path = "/test/rust_overlay" + rust_bootstrap_nix_path = "/test/bootstrap_nix_path" + configuration_nix_path = "/test/config_nix_path" + rust_src_flake_path = "/test/rust_src_path" + dry_run = true + rustc_path = "/test/rustc" + cargo_path = "/test/cargo" + rust_channel = "nightly" + rust_download_rustc = true + rust_parallel_compiler = true + rust_llvm_tools = true + rust_debuginfo_level = 2 + patch_binaries_for_nix = true + vendor = true + build_dir = "/test/build_dir" + build_jobs = 8 + home_dir = "/test/home" + cargo_home_dir = "/test/cargo_home" + install_prefix = "/test/install_prefix" + install_sysconfdir = "/test/install_sysconfdir" + dist_sign_folder = "/test/dist_sign_folder" + dist_upload_addr = "test_upload_addr" + llvm_download_ci_llvm = true + llvm_ninja = true + change_id = "test_change_id" + "#; + + let config: AppConfig = toml::from_str(toml_content).unwrap(); + + assert_eq!(config.stage, Some("test_stage".to_string())); + assert_eq!(config.target, Some("test_target".to_string())); + assert_eq!(config.project_root, Some(PathBuf::from("/test/project_root"))); + assert_eq!(config.system, Some("test_system".to_string())); + assert_eq!(config.output, Some(PathBuf::from("/test/output"))); + assert_eq!(config.rust_bootstrap_nix_flake_ref, Some("test_bootstrap_ref".to_string())); + assert_eq!(config.rust_src_flake_ref, Some("test_src_ref".to_string())); + assert_eq!(config.nixpkgs_path, Some(PathBuf::from("/test/nixpkgs"))); + assert_eq!(config.rust_overlay_path, Some(PathBuf::from("/test/rust_overlay"))); + assert_eq!(config.rust_bootstrap_nix_path, Some(PathBuf::from("/test/bootstrap_nix_path"))); + assert_eq!(config.configuration_nix_path, Some(PathBuf::from("/test/config_nix_path"))); + assert_eq!(config.rust_src_flake_path, Some(PathBuf::from("/test/rust_src_path"))); + assert_eq!(config.dry_run, Some(true)); + assert_eq!(config.rustc_path, Some(PathBuf::from("/test/rustc"))); + assert_eq!(config.cargo_path, Some(PathBuf::from("/test/cargo"))); + assert_eq!(config.rust_channel, Some("nightly".to_string())); + assert_eq!(config.rust_download_rustc, Some(true)); + assert_eq!(config.rust_parallel_compiler, Some(true)); + assert_eq!(config.rust_llvm_tools, Some(true)); + assert_eq!(config.rust_debuginfo_level, Some(2)); + assert_eq!(config.patch_binaries_for_nix, Some(true)); + assert_eq!(config.vendor, Some(true)); + assert_eq!(config.build_dir, Some(PathBuf::from("/test/build_dir"))); + assert_eq!(config.build_jobs, Some(8)); + assert_eq!(config.home_dir, Some(PathBuf::from("/test/home"))); + assert_eq!(config.cargo_home_dir, Some(PathBuf::from("/test/cargo_home"))); + assert_eq!(config.install_prefix, Some(PathBuf::from("/test/install_prefix"))); + assert_eq!(config.install_sysconfdir, Some(PathBuf::from("/test/install_sysconfdir"))); + assert_eq!(config.dist_sign_folder, Some(PathBuf::from("/test/dist_sign_folder"))); + assert_eq!(config.dist_upload_addr, Some("test_upload_addr".to_string())); + assert_eq!(config.llvm_download_ci_llvm, Some(true)); + assert_eq!(config.llvm_ninja, Some(true)); + assert_eq!(config.change_id, Some("test_change_id".to_string())); +} + +#[test] +fn test_app_config_merge_with_args() { + let mut config = AppConfig { + stage: Some("initial_stage".to_string()), + target: Some("initial_target".to_string()), + rustc_path: Some(PathBuf::from("/initial/rustc")), + ..Default::default() + }; + + let args = Args { + stage: Some("arg_stage".to_string()), + rustc_path: Some(PathBuf::from("/arg/rustc")), + vendor: Some(true), + ..Default::default() + }; + + config.merge_with_args(&args); + + assert_eq!(config.stage, Some("arg_stage".to_string())); + assert_eq!(config.target, Some("initial_target".to_string())); // Should remain unchanged + assert_eq!(config.rustc_path, Some(PathBuf::from("/arg/rustc"))); + assert_eq!(config.vendor, Some(true)); +} + +#[test] +fn test_format_file_all_placeholders() { + let template_content = r#"## +# Generated by bootstrap-config-builder +# +# System: {system} +# Project Root: {flake_path_str} + +[nix] +nixpkgs_path = "{nixpkgs_path}" +rust_overlay_path = "{rust_overlay_path}" +rust_bootstrap_nix_path = "{rust_bootstrap_nix_path}" +configuration_nix_path = "{configuration_nix_path}" +rust_src_flake_path = "{rust_src_flake_path}" +rust_bootstrap_nix_flake_ref = "{rust_bootstrap_nix_flake_ref}" +rust_src_flake_ref = "{rust_src_flake_ref}" + +[rust] +rustc = "{rustc_path}" +cargo = "{cargo_path}" +channel = "{rust_channel}" +download-rustc = {rust_download_rustc} +parallel-compiler = {rust_parallel_compiler} +llvm-tools = {rust_llvm_tools} +debuginfo-level = {rust_debuginfo_level} + +[build] +stage = {stage} +target = "{target}" +patch-binaries-for-nix = {patch_binaries_for_nix} +vendor = {vendor} +build-dir = "{build_dir}" +jobs = {build_jobs} + +[env] +HOME = "{home_dir}" +CARGO_HOME = "{cargo_home_dir}" + +[install] +prefix = "{install_prefix}" +sysconfdir = "{install_sysconfdir}" + +[dist] +sign-folder = "{dist_sign_folder}" +upload-addr = "{dist_upload_addr}" + +[llvm] +download-ci-llvm = {llvm_download_ci_llvm} +ninja = {llvm_ninja} + +[change-id] +id = "{change_id}" + "#; + + // Create a dummy example.toml file for the test + let temp_dir = tempfile::tempdir().unwrap(); + let template_path = temp_dir.path().join("example.toml"); + std::fs::write(&template_path, template_content).unwrap(); + + let result = format_file::format_file( + template_path.to_str().unwrap(), + "test_system", + "/test/flake_path", + "/test/nixpkgs", + "/test/rust_overlay", + "/test/bootstrap_nix_path", + "/test/config_nix_path", + "/test/rust_src_path", + "test_bootstrap_ref", + "test_src_ref", + "1", + "x86_64-unknown-linux-gnu", + "/test/rustc", + "/test/cargo", + "stable", + false, + true, + false, + 1, + true, + false, + "/test/build_dir", + 4, + "/test/home", + "/test/cargo_home", + "/test/install_prefix", + "/test/install_sysconfdir", + "/test/dist_sign_folder", + "test_upload_addr", + false, + true, + "test_change_id", + ); + + assert!(result.contains("System: test_system")); + assert!(result.contains("Project Root: /test/flake_path")); + assert!(result.contains("nixpkgs_path = \"/test/nixpkgs\"")); + assert!(result.contains("rust_overlay_path = \"/test/rust_overlay\"")); + assert!(result.contains("rust_bootstrap_nix_path = \"/test/bootstrap_nix_path\"")); + assert!(result.contains("configuration_nix_path = \"/test/config_nix_path\"")); + assert!(result.contains("rust_src_flake_path = \"/test/rust_src_path\"")); + assert!(result.contains("rust_bootstrap_nix_flake_ref = \"test_bootstrap_ref\"")); + assert!(result.contains("rust_src_flake_ref = \"test_src_ref\"")); + assert!(result.contains("stage = 1")); + assert!(result.contains("target = \"x86_64-unknown-linux-gnu\"")); + assert!(result.contains("rustc = \"/test/rustc\"")); + assert!(result.contains("cargo = \"/test/cargo\"")); + assert!(result.contains("channel = \"stable\"")); + assert!(result.contains("download-rustc = false")); + assert!(result.contains("parallel-compiler = true")); + assert!(result.contains("llvm-tools = false")); + assert!(result.contains("debuginfo-level = 1")); + assert!(result.contains("patch-binaries-for-nix = true")); + assert!(result.contains("vendor = false")); + assert!(result.contains("build-dir = \"/test/build_dir\"")); + assert!(result.contains("jobs = 4")); + assert!(result.contains("HOME = \"/test/home\"")); + assert!(result.contains("CARGO_HOME = \"/test/cargo_home\"")); + assert!(result.contains("prefix = \"/test/install_prefix\"")); + assert!(result.contains("sysconfdir = \"/test/install_sysconfdir\"")); + assert!(result.contains("sign-folder = \"/test/dist_sign_folder\"")); + assert!(result.contains("upload-addr = \"test_upload_addr\"")); + assert!(result.contains("download-ci-llvm = false")); + assert!(result.contains("ninja = true")); + assert!(result.contains("id = \"test_change_id\"")); +} diff --git a/build_rust_bootstrap.sh b/build_rust_bootstrap.sh new file mode 100755 index 00000000..bb216e12 --- /dev/null +++ b/build_rust_bootstrap.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -euo pipefail + +pushd /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix/standalonex/src/bootstrap/ +cargo build +popd \ No newline at end of file diff --git a/config.toml b/config.toml index 991c4831..1bfe5b59 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,20 @@ -vendor = true -rustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc" -cargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo" -HOME = "/data/data/com.termux.nix/files/usr/tmp/nix-shell.GCgkig/nix-shell.NprGk6/tmp.cHrvt4OLAq" -CARGO_HOME = "/data/data/com.termux.nix/files/usr/tmp/nix-shell.GCgkig/nix-shell.NprGk6/tmp.cHrvt4OLAq/.cargo" +[rust] +rustc = "/dummy/rustc" +cargo = "/dummy/cargo" + +[build] +rustc = "/dummy/rustc" +cargo = "/dummy/cargo" + +[env] +HOME = "/tmp/home" +CARGO_HOME = "/tmp/cargo-home" + +[git] +base_branch = "feature/CRQ-016-nixify" +new_flake_branch_prefix = "feature" +component = "solana-rust-1.84.1" +arch = "aarch64" +phase = "phase0" +step = "step1" +output_dir_prefix = "flakes" diff --git a/configuration-nix/Cargo.lock b/configuration-nix/Cargo.lock new file mode 100644 index 00000000..fe8dcabf --- /dev/null +++ b/configuration-nix/Cargo.lock @@ -0,0 +1,105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bootstrap-config-processor" +version = "0.1.0" +dependencies = [ + "bootstrap-config-utils", +] + +[[package]] +name = "bootstrap-config-utils" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", + "stage0_parser_crate", + "toml", +] + +[[package]] +name = "configuration-nix" +version = "0.1.0" + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "stage0_parser_crate" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "syn" +version = "2.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" diff --git a/configuration-nix/Cargo.toml b/configuration-nix/Cargo.toml new file mode 100644 index 00000000..88dee437 --- /dev/null +++ b/configuration-nix/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "configuration-nix" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.0", features = ["derive", "env"] } diff --git a/configuration-nix/flake.lock b/configuration-nix/flake.lock new file mode 100644 index 00000000..8029d932 --- /dev/null +++ b/configuration-nix/flake.lock @@ -0,0 +1,205 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "rustSrcFlake": "rustSrcFlake" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { + "inputs": { + "nixpkgs": "nixpkgs_4" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustSrcFlake": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_3", + "rust-overlay": "rust-overlay_2" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/configuration-nix/flake.nix b/configuration-nix/flake.nix new file mode 100644 index 00000000..ddfa2d20 --- /dev/null +++ b/configuration-nix/flake.nix @@ -0,0 +1,58 @@ +{ + description = "Nix flake for the configuration-nix Rust crate"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + flake-utils.url = "github:meta-introspector/flake-utils?ref=feature/CRQ-016-nixify"; # Corrected + rustSrcFlake.url = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, rust-overlay, flake-utils, rustSrcFlake }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ rust-overlay.overlays.default ]; + }; + rustToolchain = pkgs.rustChannels.nightly.rust.override { + targets = [ + (if system == "aarch64-linux" then "aarch64-unknown-linux-gnu" else "x86_64-unknown-linux-gnu") + ]; + }; + in + { + packages.default = pkgs.rustPlatform.buildRustPackage { + pname = "configuration-nix"; + version = "0.1.0"; + src = ./.; + cargoLock = { + lockFile = ./Cargo.lock; + }; + buildInputs = [ rustToolchain pkgs.clap ]; + }; + + apps.default = flake-utils.lib.mkApp { + drv = pkgs.writeShellScriptBin "generate-config" '' + ${self.packages.${system}.default}/bin/configuration-nix \ + --stage "$1" \ + --target "$2" \ + --nixpkgs-path "${nixpkgs.outPath}" \ + --rust-overlay-path "${rust-overlay.outPath}" \ + --configuration-nix-path "${self.outPath}" \ + --rust-src-flake-path "${rustSrcFlake.outPath}" \ + --rust-bootstrap-nix-flake-ref "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify" \ + --rust-src-flake-ref "github:meta-introspector/rust?ref=feature/CRQ-016-nixify" + ''; + }; + + devShells.default = pkgs.mkShell { + packages = [ + pkgs.cargo + pkgs.rustc + pkgs.nix + ]; + }; + } + ); +} diff --git a/configuration-nix/src/config_generator.rs b/configuration-nix/src/config_generator.rs new file mode 100644 index 00000000..2b35a8f4 --- /dev/null +++ b/configuration-nix/src/config_generator.rs @@ -0,0 +1,32 @@ +// configuration-nix/src/config_generator.rs + +use crate::config_params::ConfigParams; +use std::fs; + +pub fn generate_config_toml(params: &ConfigParams) { + + // Construct config.toml content + let config_content = format!( + r###"# Generated by configuration-nix + +[nix] +nixpkgs_path = "{}" +rust_overlay_path = "{}" +configuration_nix_path = "{}" +rust_src_flake_path = "{}" + +[build] +stage = {} +target = "{}" +"###, + params.nixpkgs_path.to_string_lossy(), + params.rust_overlay_path.to_string_lossy(), + params.configuration_nix_path.to_string_lossy(), + params.rust_src_flake_path.to_string_lossy(), + params.stage, + params.target + ); + + let config_file_path = "config.toml".to_string(); + fs::write(&config_file_path, config_content).unwrap(); +} diff --git a/configuration-nix/src/config_params.rs b/configuration-nix/src/config_params.rs new file mode 100644 index 00000000..e02e259d --- /dev/null +++ b/configuration-nix/src/config_params.rs @@ -0,0 +1,41 @@ +use clap::Parser; +use std::path::PathBuf; + +/// Struct to hold all configuration parameters for generating config.toml. +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct ConfigParams { + /// The bootstrap stage number (e.g., 0, 1, 2) + #[arg()] + pub stage: String, + + /// The target triple for the build (e.g., aarch64-unknown-linux-gnu) + #[arg()] + pub target: String, + + /// Path to the nixpkgs flake input + #[arg(long)] + pub nixpkgs_path: PathBuf, + + /// Path to the rust-overlay flake input + #[arg(long)] + pub rust_overlay_path: PathBuf, + + + + /// Path to the configurationNix flake input + #[arg(long)] + pub configuration_nix_path: PathBuf, + + /// Path to the rustSrcFlake input + #[arg(long)] + pub rust_src_flake_path: PathBuf, + + /// The flake reference for the rust-bootstrap-nix repository + #[arg(long)] + pub rust_bootstrap_nix_flake_ref: String, + + /// The flake reference for the rust source + #[arg(long)] + pub rust_src_flake_ref: String, +} diff --git a/configuration-nix/src/main.rs b/configuration-nix/src/main.rs new file mode 100644 index 00000000..7c7d29a1 --- /dev/null +++ b/configuration-nix/src/main.rs @@ -0,0 +1,9 @@ +use clap::Parser; + +mod config_generator; +mod config_params; + +fn main() { + let params = config_params::ConfigParams::parse(); + config_generator::generate_config_toml(¶ms); +} diff --git a/configv2.toml b/configv2.toml new file mode 100644 index 00000000..7675161a --- /dev/null +++ b/configv2.toml @@ -0,0 +1,11 @@ +[rust] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo" + +[build] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo" + +[env] +HOME = "/tmp/home" +CARGO_HOME = "/tmp/cargo-home" diff --git a/create_stage0_env.sh b/create_stage0_env.sh new file mode 100755 index 00000000..62126186 --- /dev/null +++ b/create_stage0_env.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Host triple +HOST_TRIPLE="aarch64-unknown-linux-gnu" + +# Create build directory structure +mkdir -p build/${HOST_TRIPLE}/stage0 +mkdir -p build/${HOST_TRIPLE}/stage0-sysroot/lib + +# Environment variables from run_rustc_shim.sh +export RUSTC_STAGE=0 +export RUSTC_SNAPSHOT="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu/bin/rustc" +export RUSTC_SYSROOT="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu" +export RUSTC_SNAPSHOT_LIBDIR="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu/lib" +export LD_LIBRARY_PATH="/nix/store/x9w1w2c9rycrdkp3ynmwjkyk2v40vyb0-get-libdir-1-84-1" +export RUST_BACKTRACE=full +export LD_DEBUG=files + +# Run the bootstrap binary with --src pointing to standalonex +/nix/store/8f16fw6m01bqhzlbcc8flvj9y3fh6bhw-bootstrap-0.1.0/bin/bootstrap check --src /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix/standalonex diff --git a/debug_build.sh b/debug_build.sh index 4a2371bd..8f8aa1a8 100755 --- a/debug_build.sh +++ b/debug_build.sh @@ -7,11 +7,14 @@ echo "" echo "which curl:" which curl echo "" -echo "--- Creating config.toml ---" -echo "patch-binaries-for-nix = true" > config.toml -echo "vendor = true" >> config.toml -echo "rustc = \"$(which rustc)\"" >> config.toml -echo "cargo = \"$(which cargo)\"" >> config.toml +echo "--- Creating config.toml using bootstrap-config-builder ---" +./bootstrap-config-builder/target/debug/bootstrap-config-builder \ + --project-root "$(pwd)" \ + --rustc-path "$(which rustc)" \ + --cargo-path "$(which cargo)" \ + --patch-binaries-for-nix true \ + --vendor true \ + --output config.toml echo "" echo "cat config.toml:" cat config.toml diff --git a/docs/Config_TOML_Reference.md b/docs/Config_TOML_Reference.md new file mode 100644 index 00000000..92d954e2 --- /dev/null +++ b/docs/Config_TOML_Reference.md @@ -0,0 +1,158 @@ +# config.toml Reference + +This document provides a comprehensive reference for all configurable options within the `config.toml` file used by the `rust-bootstrap-nix` project. This file is central to defining the build environment, toolchain paths, and various build-time behaviors. + +## Overview + +The `config.toml` file is generated by the `bootstrap-config-builder` tool, which can take inputs from command-line arguments or a base configuration file. It allows for fine-grained control over the Rust bootstrap process, ensuring reproducibility and adaptability across different environments, especially within Nix. + +## Sections and Options + +### `[nix]` Section + +This section defines paths and references related to the Nix ecosystem. + +* `nixpkgs_path` (Path): The absolute path to the `nixpkgs` source. + * **Purpose:** Specifies the version and location of the Nix Packages collection to be used for dependencies. + * **Example:** `nixpkgs_path = "/nix/store/...-nixpkgs"` + +* `rust_overlay_path` (Path): The absolute path to the Rust overlay source. + * **Purpose:** Specifies the location of a custom Nix overlay for Rust, which might contain specialized Rust toolchains or packages. + * **Example:** `rust_overlay_path = "/nix/store/...-rust-overlay"` + +* `rust_bootstrap_nix_path` (Path): The absolute path to the `rust-bootstrap-nix` project root. + * **Purpose:** Defines the root directory of the current project, used for resolving relative paths. + +* `configuration_nix_path` (Path): The absolute path to the `configuration-nix` source. + * **Purpose:** Specifies the location of the `configuration-nix` flake, which might be used for generating additional configuration. + +* `rust_src_flake_path` (Path): The absolute path to the `rust-src` flake source. + * **Purpose:** Points to the source code of the Rust compiler, which is a foundational input for the bootstrap process. + +* `rust_bootstrap_nix_flake_ref` (String): The flake reference for `rust-bootstrap-nix`. + * **Purpose:** Specifies the Git reference (e.g., branch, tag, commit hash) for the `rust-bootstrap-nix` flake input. + * **Example:** `rust_bootstrap_nix_flake_ref = "github:meta-introspector/rust-bootstrap-nix?ref=main"` + +* `rust_src_flake_ref` (String): The flake reference for `rust-src`. + * **Purpose:** Specifies the Git reference for the `rust-src` flake input. + * **Example:** `rust_src_flake_ref = "github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2"` + +### `[rust]` Section + +This section configures the Rust toolchain and its behavior. + +* `rustc` (Path): The absolute path to the `rustc` (Rust compiler) executable. + * **Purpose:** Ensures that the build uses a precisely defined and versioned compiler. + * **Example:** `rustc = "/nix/store/.../bin/rustc"` + +* `cargo` (Path): The absolute path to the `cargo` (Rust package manager) executable. + * **Purpose:** Guarantees the use of a specific, Nix-managed `cargo` instance. + * **Example:** `cargo = "/nix/store/.../bin/cargo"` + +* `channel` (String): The Rust release channel to use. + * **Purpose:** Specifies whether to use `stable`, `beta`, or `nightly` Rust toolchains. + * **Default:** `stable` + * **Example:** `channel = "nightly"` + +* `download-rustc` (Boolean): Whether to download `rustc`. + * **Purpose:** Controls if the `rustc` compiler should be downloaded or if an existing one should be used. + * **Default:** `false` + +* `parallel-compiler` (Boolean): Whether to enable parallel compilation. + * **Purpose:** Optimizes build times by allowing `rustc` to compile multiple crates in parallel. + * **Default:** `false` + +* `llvm-tools` (Boolean): Whether to enable LLVM tools. + * **Purpose:** Includes LLVM utilities in the build, which might be required for certain Rust features or targets. + * **Default:** `false` + +* `debuginfo-level` (Integer): The debug information level for Rust compilation. + * **Purpose:** Controls the amount of debug information generated, affecting binary size and debuggability. + * **Default:** `0` (no debug info) + * **Example:** `debuginfo-level = 1` + +### `[build]` Section + +This section defines general build-related settings. + +* `stage` (String): The build stage to configure. + * **Purpose:** Used in multi-stage build processes to specify which stage is being targeted (e.g., "0", "1", "2"). + +* `target` (String): The target triple for the build. + * **Purpose:** Specifies the architecture, vendor, operating system, and ABI for which the code is being compiled. + * **Example:** `target = "aarch64-unknown-linux-gnu"` + +* `patch-binaries-for-nix` (Boolean): Enables Nix-specific patching of binaries. + * **Purpose:** Essential for ensuring that compiled artifacts are relocatable within the Nix store, often involving adjustments to RPATHs. + * **Default:** `false` + +* `vendor` (Boolean): Enables vendoring for the Rust build process. + * **Purpose:** Ensures that dependencies are present locally (e.g., in a `vendor/` directory) rather than being downloaded during the build, crucial for reproducible Nix builds. + * **Default:** `false` + +* `build-dir` (Path): The directory where build artifacts are placed. + * **Purpose:** Overrides the default build output directory. + +* `jobs` (Integer): The number of parallel build jobs. + * **Purpose:** Controls the concurrency of the build process, often set to the number of CPU cores. + * **Default:** `0` (auto-detect) + +### `[env]` Section + +This section defines environment variables for the build process. + +* `HOME` (Path): The `HOME` directory for the build. + * **Purpose:** Sets the `HOME` environment variable to a temporary, isolated directory to prevent pollution of the user's actual home directory. + +* `CARGO_HOME` (Path): The `CARGO_HOME` directory for the build. + * **Purpose:** Ensures that Cargo's caches, registries, and other state are kept isolated within the build environment. + +### `[install]` Section + +This section controls where the built artifacts will be placed. + +* `prefix` (Path): The base directory for all installed components. + * **Purpose:** In a Nix environment, this will typically be a path within the Nix store. All other installation paths (like `bindir`, `libdir`, etc.) will be derived from this prefix unless explicitly overridden. + * **Example:** `prefix = "/nix/store/some-hash-my-rust-package"` + +* `sysconfdir` (Path): The directory for system-wide configuration files. + * **Purpose:** Specifies where configuration files should be installed. + +### `[dist]` Section + +This section configures distribution-related settings. + +* `sign-folder` (Path): The folder for distribution signing. + * **Purpose:** Specifies the directory where distribution artifacts are signed. + +* `upload-addr` (String): The upload address for distribution. + * **Purpose:** Defines the target address for uploading built distributions. + +### `[llvm]` Section + +This section configures LLVM-related settings. + +* `download-ci-llvm` (Boolean): Whether to download CI LLVM. + * **Purpose:** Controls if LLVM components from continuous integration should be downloaded. + * **Default:** `false` + +* `ninja` (Boolean): Whether to use Ninja for LLVM builds. + * **Purpose:** Specifies if the Ninja build system should be used for compiling LLVM. + * **Default:** `false` + +### `[target.]` Section (Example) + +This is an example of a target-specific section, where `` would be a target triple like `aarch64-unknown-linux-gnu`. + +* `cc` (Path): The path to the C compiler for this target. + * **Purpose:** Allows specifying a custom C compiler for a particular target. + +* `android-ndk` (Path): The path to the Android NDK for this target. + * **Purpose:** Specifies the Android NDK location when building for Android targets. + +### `[change-id]` Section + +This section is used for tracking major changes in the configuration. + +* `id` (String): A unique identifier for tracking configuration changes. + * **Purpose:** Helps in identifying and managing different versions or significant modifications of the `config.toml`. diff --git a/docs/Nix_Integration.md b/docs/Nix_Integration.md new file mode 100644 index 00000000..5b84581d --- /dev/null +++ b/docs/Nix_Integration.md @@ -0,0 +1,53 @@ +# Nix Integration + +This document provides a detailed explanation of the Nix integration within the `rust-bootstrap-nix` project. This system is designed to create a reproducible and flexible build environment by dynamically resolving Nix store paths for all the tools and dependencies required for the Rust bootstrap process. + +## The `generated_config.toml` File + +The core of this system is the `generated_config.toml` file. This file is generated by the `bootstrap-config-builder` utility and contains the exact Nix store paths for all the tools and dependencies required for the build, including: + +* `rustc`: The Rust compiler. +* `cargo`: The Rust package manager. +* `nixpkgs`: The Nix Packages collection. +* `rust-overlay`: A Nix overlay for Rust. +* `rust-src-flake`: The Rust source code. + +Here's an example of a `generated_config.toml` file: + +```toml +# Generated by bootstrap-config-builder +# +# System: aarch64-linux +# Project Root: /path/to/rust-bootstrap-nix + +[nix] +nixpkgs_path = "/nix/store/..." +rust_overlay_path = "/nix/store/..." +rust_bootstrap_nix_path = "/path/to/rust-bootstrap-nix" +configuration_nix_path = "/path/to/rust-bootstrap-nix/configuration-nix" +rust_src_flake_path = "/nix/store/..." +rust_bootstrap_nix_flake_ref = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify" +rust_src_flake_ref = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify" + +[rust] +rustc = "/nix/store/.../bin/rustc" +cargo = "/nix/store/.../bin/cargo" +# ... +``` + +## Dynamic Path Resolution + +The Rust bootstrap process has been modified to read this `generated_config.toml` file and use the paths within it to configure the build. If the `generated_config.toml` file is not present, the bootstrap process will dynamically fetch the required Nix store paths using `nix flake prefetch` and `nix path-info`. + +This is achieved through the `resolve_nix_paths` function in the `Config` struct, which is called during the bootstrap process. This function checks if the Nix paths are already set in the `Config` struct (from `generated_config.toml`). If not, it executes the necessary Nix commands to resolve the paths and updates the `Config` struct accordingly. + +## Usage + +To use this new system, you can either: + +1. **Generate `generated_config.toml` manually:** Run the `bootstrap-config-builder` with the desired `--rustc-path`, `--cargo-path`, and `--rust-src-flake-path` arguments. This gives you fine-grained control over the exact versions of the tools and dependencies used in the build. +2. **Let the bootstrap process handle it:** If `generated_config.toml` is not present, the bootstrap process will automatically resolve the Nix paths for you. This is a convenient way to get up and running quickly, but it provides less control over the exact versions used. + +## Lattice of Nix Flakes + +This dynamic, Nix-based configuration system is the foundation for creating a "lattice of Nix flakes." This allows us to precisely control and track every component of the Rust bootstrap process, from compiler stages to individual Cargo runs, all within the Nix ecosystem. This is a powerful tool for reproducible builds and toolchain experimentation, and it's a key step towards achieving the 8-fold recursion system for eBPF Rust bootstrapping. diff --git a/docs/Nix_Rust_Integration_Process.md b/docs/Nix_Rust_Integration_Process.md new file mode 100644 index 00000000..5227da3d --- /dev/null +++ b/docs/Nix_Rust_Integration_Process.md @@ -0,0 +1,45 @@ +# Nix-Rust Integration Process: Passing Flake URLs via config.toml + +## 1. Goal + +The primary objective is to deeply integrate the Rust project's build process with the Nix ecosystem. This involves using Nix flake URLs as inputs to the Rust build system, enabling Rust to query Nix for build information, and ultimately building a specific version of Rust (e.g., from `github:meta-introspector/rust?ref=feature/CRQ-016-nixify`) through an 8-level bootstrap process managed by Nix. + +## 2. `bootstrap-config-builder` Role: Generating `config.toml` + +The `bootstrap-config-builder` is a Rust utility responsible for generating the `config.toml` file, which serves as the primary configuration input for the Rust bootstrap process. This builder is now capable of incorporating Nix flake URLs and paths into the generated `config.toml`. + +### Key Mechanisms: + +* **`example.toml` Template:** The builder uses `bootstrap-config-builder/src/example.toml` as a template. This template defines the structure of the output `config.toml` and includes named placeholders (e.g., `{system}`, `{nixpkgs_path}`, `{rust_src_flake_ref}`) for dynamic values. + +* **`format_file` Function:** The `bootstrap-config-builder/src/utils/format_file.rs` module contains the `format_file` function, which reads the `example.toml` template and replaces its placeholders with actual values provided via command-line arguments or a configuration file. + +* **Configuration Loading and Overrides:** The `bootstrap-config-builder` now supports loading configuration from an external `config.toml` file (specified via `--config-file`) and merging it with command-line arguments. Command-line arguments take precedence, allowing for flexible overrides. + * **`bootstrap-config-builder/src/args.rs`**: Defines command-line arguments, including `config_file` and various Nix-related paths and flake references. + * **`bootstrap-config-builder/src/config.rs`**: Defines the `AppConfig` struct, which can be deserialized from a TOML file and includes a `merge_with_args` method to apply command-line overrides. + * **`bootstrap-config-builder/src/main.rs`**: Orchestrates the parsing of CLI arguments, loading of `config.toml`, merging configurations, and then using the final `AppConfig` to generate the output `config.toml` content. + +## 3. Rust Bootstrap (`standalonex`) Configuration Consumption + +The `standalonex/src/bootstrap` component of the Rust project is being refactored to consume the `config.toml` generated by `bootstrap-config-builder`. This enables the Rust bootstrap to understand and utilize the Nix-related configuration. + +### Key Components for Consumption: + +* **`LocalTomlConfig` (`standalonex/src/bootstrap/src/core/config_utils/src/local_toml_config.rs`):** This struct represents the overall structure of the `config.toml` file as understood by the Rust bootstrap. It now includes a `nix: Option` field to specifically capture the Nix-related configuration. + +* **`LocalNixConfig` (`standalonex/src/bootstrap/src/core/config_utils/src/local_nix_config.rs`):** A newly introduced struct that mirrors the `[nix]` section of the `config.toml`. It contains fields for `nixpkgs_path`, `rust_overlay_path`, `rust_bootstrap_nix_path`, `configuration_nix_path`, `rust_src_flake_path`, `rust_bootstrap_nix_flake_ref`, and `rust_src_flake_ref`. + +* **`NixConfigApplicator` (`standalonex/src/bootstrap/src/core/config_utils/src/nix_config.rs`):** This is a `ConfigApplicator` implementation responsible for taking the deserialized `LocalNixConfig` values and applying them to the `ParsedConfig` struct, which holds the final, merged configuration for the Rust bootstrap. + +* **`ParsedConfig` (`standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs`):** This central struct now includes dedicated fields to store the Nix-related paths and flake references, making them accessible throughout the Rust bootstrap logic. + +* **`parse.rs` (`standalonex/src/bootstrap/src/core/config_utils/src/parse.rs`):** This module orchestrates the configuration parsing. It now includes `NixConfigApplicator` in its list of applicators, ensuring that the Nix configuration is processed and applied during the bootstrap's configuration phase. + +## 4. Next Steps (High-Level) + +With the `config.toml` generation and consumption mechanisms in place, the subsequent steps will focus on enabling the Rust bootstrap to actively interact with Nix: + +1. **Nix Interaction from Rust:** Implement logic within the Rust bootstrap to execute Nix commands (e.g., `nix eval`, `nix build`) using the paths and flake references obtained from `ParsedConfig`. +2. **Parsing Nix Output:** Process the output of Nix commands to extract necessary information (e.g., resolved Nix store paths for dependencies, build artifacts). +3. **Integration into Rust Build:** Integrate the resolved Nix information into the existing Rust 8-level bootstrap build process, ensuring that the Rust compiler and its components are built using Nix-managed inputs. +4. **Flake Generation:** Develop a mechanism to generate a new Nix flake or a virtual flake that encapsulates the entire Rust build process, making it fully reproducible and Nix-driven. \ No newline at end of file diff --git a/docs/QA_Plan.md b/docs/QA_Plan.md new file mode 100644 index 00000000..b3221ed3 --- /dev/null +++ b/docs/QA_Plan.md @@ -0,0 +1,40 @@ +# QA Plan for rust-bootstrap-nix Project + +This document outlines the Quality Assurance plan for the `rust-bootstrap-nix` project, focusing on key areas and testing methodologies. + +## 1. Identify Key Test Areas: + +* **`bootstrap-config-builder`:** + * Configuration loading, merging, and override logic. + * Precondition checks (`preconditions.rs`). + * `nix-dir` tool functionality (output, filtering, JSON). +* **Rust Bootstrap Integration with Nix:** + * Correct reading and utilization of Nix configuration from `ParsedConfig`. + * Accurate execution of Nix commands from Rust (e.g., `nix eval`, `nix build`). + * Successful integration of resolved Nix store paths into the Rust build process. + * Verification of the 8-level bootstrap process. +* **Overall Project Build & Test Commands:** + * Functionality of the main `Makefile` targets. + * Correctness of `nix-build` and `nix-shell` commands. +* **`devShell` Environment:** + * Ensuring all necessary development tools are available and correctly configured. + +## 2. Determine Test Types and Tools: + +* **Unit Tests:** Leverage Rust's built-in `cargo test` for individual functions and modules, especially within `bootstrap-config-builder` and the refactored `standalonex/src/bootstrap/src/lib.rs` components. +* **Integration Tests:** + * **Rust-Nix Interaction:** Write Rust tests that call the Nix interaction logic and assert on the outcomes (e.g., correct Nix command execution, valid path resolution). + * **Component Interaction:** Test the flow between `bootstrap-config-builder` and the main Rust bootstrap process. +* **System/End-to-End Tests:** + * **Shell Scripts:** Create or enhance existing shell scripts (`test.sh`, `run_bootstrap_test.sh`, etc.) to execute the full bootstrap process and verify the final output (e.g., successful compilation, correct artifacts). + * **Nix Checks:** Use `nix-build --check` and potentially `nix-diff` to ensure flake outputs are consistent and correct. +* **Static Analysis & Linting:** Ensure pre-commit hooks (`.pre-commit-config.yaml`) are comprehensive and run regularly. This includes `rustfmt`, `clippy`, and potentially `shellcheck` for shell scripts. +* **Documentation Review:** Regularly verify that `docs/` accurately reflects the current state and functionality of the project. + +## 3. Proposed Next Steps for Implementation: + +* **Review Existing Tests:** Identify current unit, integration, and system tests. +* **Prioritize Test Cases:** Based on the `TODO.md`, focus on critical paths first (e.g., `bootstrap-config-builder` output, core Nix integration). +* **Expand Unit Test Coverage:** For new and refactored Rust code. +* **Develop Integration Tests:** Specifically for the Rust-Nix interface. +* **Enhance End-to-End Scripts:** To cover the full build process and verify outputs. diff --git a/docs/README_bootstrap_builder_flake.md b/docs/README_bootstrap_builder_flake.md new file mode 100644 index 00000000..714a9cce --- /dev/null +++ b/docs/README_bootstrap_builder_flake.md @@ -0,0 +1,117 @@ +# Bootstrap Builder Flake + +This flake is responsible for building the Rust bootstrap compiler from source. + +## Overall Goal: +Automate the generation, Nix build, and Git management of new flakes, ensuring each step is versioned and traceable within a lattice-structured branching model. + +## Current Status: +* `flake-template-generator` is updated to accept lattice-based branch naming components. +* The Nix build step within `flake-template-generator` has been removed to prevent circular dependencies. +* Git operations within `flake-template-generator` are now configured to use a configurable base branch. +* The repository is currently clean, and previous untracked files have been removed. + +## Detailed Plan: + +**Overall Goal:** Automate the generation, Nix build, and Git management of new flakes, ensuring versioning and traceability within a lattice-structured branching model for `solana-rust-1.84.1`. + +**Current State:** +* `flake-template-generator` has been refactored into modular components (`args.rs`, `config_parser.rs`, `flake_generator.rs`, `file_writer.rs`, `statix_checker.rs`). +* The `run_git_command` function and Git operations have been completely removed from `flake-template-generator`. +* A new Git utility function, `create_and_push_branch`, exists in `standalonex/src/bootstrap/src/core/generate_steps/git_modules/create_branch.rs`. +* The `flake-template-generator/Makefile` has been updated to `COMPONENT := solana-rust-1.84.1`. +* The current Git branch is `feature/CRQ-016-nixify`. + +**Detailed Plan:** + +**Phase 1: Prepare the Environment and Generate the Flake** + +1. **Ensure Clean Git State (Verification):** + * Run `git status` to confirm no uncommitted changes on `feature/CRQ-016-nixify`. + * If there are uncommitted changes, either commit them or stash them. + +2. **Generate the Flake Files for `solana-rust-1.84.1`:** + * **Action:** Execute `make -C flake-template-generator generate-flake`. + * **Expected Outcome:** This will generate `flake.nix` and `config.toml` in the directory `flakes/solana-rust-1.84.1/aarch64/phase0/step1/` (relative to the project root). The `flake.nix` will use `pkgs.writeText` and the `config.toml` will reflect the `solana-rust-1.84.1` component. + * **Verification:** Check for the existence of the generated files and their content. + +**Phase 2: Git Management using `create_and_push_branch`** + +1. **Create a Temporary Rust Binary to Call `create_and_push_branch`:** + * **Action:** Create a new directory `temp_git_runner` at the project root. + * **Action:** Create `temp_git_runner/Cargo.toml` with `standalonex` as a path dependency. + ```toml + [package] + name = "temp_git_runner" + version = "0.1.0" + edition = "2021" + + [dependencies] + standalonex = { path = "../../standalonex" } + ``` + * **Action:** Create `temp_git_runner/src/main.rs` with the following content: + ```rust + use standalonex::bootstrap::src::core::generate_steps::git_modules::create_branch::create_and_push_branch; + use std::path::PathBuf; + + fn main() -> Result<(), Box> { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); // Adjust path to point to the main project root + + let base_branch = "feature/CRQ-016-nixify".to_string(); + let new_branch = "feature/solana-rust-1.84.1/aarch64/phase0/step1".to_string(); + let generated_files_path = "flakes/solana-rust-1.84.1/aarch64/phase0/step1".to_string(); + let commit_message = format!("feat: Generated seed flake {}", new_branch); + + create_and_push_branch( + &repo_root, + &base_branch, + &new_branch, + &generated_files_path, + &commit_message, + false, // dry_run: set to true for testing, false for actual execution + false, // verbose + )?; + + Ok(()) + } + ``` + * **Verification:** Confirm the files are created correctly. + +2. **Build and Run the Temporary Git Runner:** + * **Action:** Execute `cargo run --manifest-path temp_git_runner/Cargo.toml`. + * **Expected Outcome:** The `create_and_push_branch` function will: + * Checkout `feature/CRQ-016-nixify`. + * Check if `feature/solana-rust-1.84.1/aarch64/phase0/step1` exists. + * If not, create and checkout `feature/solana-rust-1.84.1/aarch64/phase0/step1`. + * Add the generated files from `flakes/solana-rust-1.84.1/aarch64/phase0/step1`. + * Commit the changes with the specified message. + * Push the new branch to `origin`. + * **Verification:** + * Run `git status` to confirm the current branch is `feature/solana-rust-1.84.1/aarch64/phase0/step1` and there are no uncommitted changes. + * Run `git log -1` to verify the commit message. + * (Optional) Check the remote repository to confirm the new branch exists. + +3. **Clean Up the Temporary Git Runner:** + * **Action:** Remove the `temp_git_runner` directory: `rm -rf temp_git_runner`. + * **Verification:** Confirm the directory is deleted. + +**Phase 3: Nix Build of the Generated Flake** + +1. **Build the Generated Flake:** + * **Action:** Navigate to the generated flake directory: `cd flakes/solana-rust-1.84.1/aarch64/phase0/step1/`. + * **Action:** Execute `nix build .#default`. + * **Expected Outcome:** The Nix build should succeed, producing a `result` symlink to the built `config.toml`. + * **Verification:** Check for the `result` symlink and its content. + +**Phase 4: Integrate Nix Build into `standalonex` (Future Step - Not part of this detailed plan, but noted for context)** + +* Modify the `standalonex` bootstrap process to call `run_nix_build` with the generated flake path. + +**Phase 5: Document Findings and Next Steps (Future Step - Not part of this detailed plan, but noted for context)** + +* Record issues, successes, or observations; outline subsequent Nixification phases. \ No newline at end of file diff --git a/standalonex/src/build_helper/README.md b/docs/README_build_helper.md similarity index 77% rename from standalonex/src/build_helper/README.md rename to docs/README_build_helper.md index f81b631c..b5f41dec 100644 --- a/standalonex/src/build_helper/README.md +++ b/docs/README_build_helper.md @@ -1 +1,3 @@ +## build_helper + Types and functions shared across tools in this workspace. diff --git a/docs/README_build_standalone.md b/docs/README_build_standalone.md new file mode 100644 index 00000000..b31b83dc --- /dev/null +++ b/docs/README_build_standalone.md @@ -0,0 +1,9 @@ +## Building the Standalone Bootstrap + +To build the standalone Rust bootstrap environment, which is particularly useful for "Nix on Droid" (aarch64-linux) environments, use the following Nix command: + +```bash +nix build ./standalonex#packages.aarch64-linux.default +``` + +This command will build the default package defined within the `standalonex/flake.nix` for the `aarch64-linux` architecture. diff --git a/docs/README_config_binary_patching.md b/docs/README_config_binary_patching.md new file mode 100644 index 00000000..41360231 --- /dev/null +++ b/docs/README_config_binary_patching.md @@ -0,0 +1,7 @@ +### Nix-Specific Binary Patching + +The `[build]` section also includes a relevant option for Nix: + +* `patch-binaries-for-nix`: + * **Purpose:** This boolean option enables Nix-specific patching of binaries. This is essential for ensuring that compiled artifacts are truly relocatable within the Nix store, often involving adjustments to RPATHs and other internal paths. + * **Example:** `patch-binaries-for-nix = true` diff --git a/docs/README_config_docs_intro.md b/docs/README_config_docs_intro.md new file mode 100644 index 00000000..8f742115 --- /dev/null +++ b/docs/README_config_docs_intro.md @@ -0,0 +1,3 @@ +# Configuration Documentation + +This document details the various configuration files used within the `rust-bootstrap-nix` repository, primarily focusing on `config.toml` files that influence the Rust build process and environment setup. diff --git a/docs/README_config_example.md b/docs/README_config_example.md new file mode 100644 index 00000000..80b29388 --- /dev/null +++ b/docs/README_config_example.md @@ -0,0 +1,14 @@ +### Example `config.toml` for Relocatable Nix Builds + +```toml +# config.toml +[install] +prefix = "/nix/store/some-hash-my-rust-package" +# bindir will automatically be set to "/nix/store/some-hash-my-rust-package/bin" +# libdir = "lib" # would resolve to /nix/store/some-hash-my-rust-package/lib + +[build] +patch-binaries-for-nix = true +``` + +This configuration ensures that your Rust project builds and installs in a manner compatible with Nix's strict path requirements, promoting reproducibility and relocatability. diff --git a/docs/README_config_install_paths.md b/docs/README_config_install_paths.md new file mode 100644 index 00000000..a89fe4f4 --- /dev/null +++ b/docs/README_config_install_paths.md @@ -0,0 +1,3 @@ +## Configuring Relocatable Installation Paths for Nix + +For Nix-based builds and to ensure the resulting artifacts are relocatable, it's crucial to properly configure the installation paths. The `[install]` section in your `config.toml` allows you to define a base prefix for all installed components. diff --git a/docs/README_config_install_section.md b/docs/README_config_install_section.md new file mode 100644 index 00000000..47b10fe4 --- /dev/null +++ b/docs/README_config_install_section.md @@ -0,0 +1,16 @@ +### `[install]` Section + +This section controls where the built artifacts will be placed. + +* `prefix`: + * **Purpose:** Specifies the base directory for all installed components. In a Nix environment, this will typically be a path within the Nix store (e.g., `/nix/store/...-rust-toolchain`). All other installation paths (like `bindir`, `libdir`, etc.) will be derived from this prefix unless explicitly overridden. + * **Example:** `prefix = "/nix/store/some-hash-my-rust-package"` + +* `bindir`: + * **Purpose:** Specifies the directory for executable binaries. + * **Behavior:** If `prefix` is set and `bindir` is *not* explicitly defined, `bindir` will automatically default to `prefix/bin`. This ensures that your executables are placed correctly within the specified installation prefix. + * **Example (explicitly set):** `bindir = "/usr/local/bin"` (overrides the default `prefix/bin`) + +* `libdir`, `sysconfdir`, `docdir`, `mandir`, `datadir`: + * **Purpose:** These fields specify directories for libraries, configuration files, documentation, manual pages, and data files, respectively. + * **Behavior:** If `prefix` is set, these paths are typically expected to be relative to the `prefix` unless an absolute path is provided. diff --git a/docs/README_config_root.md b/docs/README_config_root.md new file mode 100644 index 00000000..178ca6e0 --- /dev/null +++ b/docs/README_config_root.md @@ -0,0 +1,20 @@ +## 1. Root `config.toml` + +**File Path:** `/config.toml` + +**Description:** This is the primary configuration file for the overall `rust-bootstrap-nix` environment. It explicitly defines how the Rust toolchain is sourced and how the build environment is isolated. + +**Key Settings:** + +* `vendor = true`: + * **Purpose:** Enables vendoring for the Rust build process. This means that dependencies are expected to be present locally (e.g., in a `vendor/` directory) rather than being downloaded from the internet during the build. This is crucial for reproducible builds in a Nix environment. +* `rustc = "/nix/store/.../bin/rustc"`: + * **Purpose:** Specifies the absolute path to the `rustc` (Rust compiler) executable within the Nix store. This ensures that the build uses a precisely defined and versioned compiler provided by Nix. +* `cargo = "/nix/store/.../bin/cargo"`: + * **Purpose:** Specifies the absolute path to the `cargo` (Rust package manager) executable within the Nix store. Similar to `rustc`, this guarantees the use of a specific, Nix-managed `cargo` instance. +* `HOME = "/data/data/com.termux.nix/files/usr/tmp/..."`: + * **Purpose:** Sets the `HOME` environment variable to a temporary, isolated directory. This prevents the build process from interacting with or polluting the user's actual home directory. +* `CARGO_HOME = "/data/data/com.termux.nix/files/usr/tmp/.../.cargo"`: + * **Purpose:** Sets the `CARGO_HOME` environment variable to a temporary `.cargo` directory. This ensures that Cargo's caches, registries, and other state are kept isolated within the build environment. + +**Overall Purpose:** The root `config.toml` is fundamental for establishing a hermetic and reproducible Rust build environment. It explicitly directs the build system to use Nix-provided tools and to operate within a clean, temporary workspace. diff --git a/docs/README_config_standalonex.md b/docs/README_config_standalonex.md new file mode 100644 index 00000000..4eaa3472 --- /dev/null +++ b/docs/README_config_standalonex.md @@ -0,0 +1,14 @@ +## 2. `standalonex/config.toml` + +**File Path:** `/standalonex/config.toml` + +**Description:** This configuration file is specific to the `standalonex` component, which is a standalone environment for the `x.py` build system. It defines the Rust toolchain paths that `x.py` should use within this isolated context. + +**Key Settings:** + +* `rustc = "/nix/store/.../bin/rustc"`: + * **Purpose:** Similar to the root `config.toml`, this specifies the absolute path to the `rustc` executable, ensuring that the `standalonex` environment uses a Nix-provided compiler. +* `cargo = "/nix/store/.../bin/cargo"`: + * **Purpose:** Specifies the absolute path to the `cargo` executable for the `standalonex` environment, guaranteeing the use of a specific, Nix-managed `cargo` instance. + +**Overall Purpose:** This `config.toml` ensures that the `standalonex` build environment, particularly when running `x.py`, is correctly configured with the appropriate Nix-provided Rust toolchain binaries. diff --git a/docs/README_config_standalonex_old.md b/docs/README_config_standalonex_old.md new file mode 100644 index 00000000..66bd30c6 --- /dev/null +++ b/docs/README_config_standalonex_old.md @@ -0,0 +1,7 @@ +## 3. `standalonex/config.old.toml` + +**File Path:** `/standalonex/config.old.toml` + +**Description:** This file appears to be an older or template version of `standalonex/config.toml`. It is specifically used by the `standalonex/flake.nix`'s `buildPhase` as a base to generate the active `config.toml` by injecting the correct Nix store paths for `rustc` and `cargo` using `sed`. + +**Purpose:** To serve as a template for generating the runtime `config.toml` within the `standalonex` build process, allowing for dynamic injection of Nix-specific paths. diff --git a/docs/README_flake_lattice.md b/docs/README_flake_lattice.md new file mode 100644 index 00000000..f77e9be9 --- /dev/null +++ b/docs/README_flake_lattice.md @@ -0,0 +1,88 @@ +# Rust-Driven Nix Flake Generation: Building a Flake Lattice + +This document outlines the plan to dynamically generate Nix flakes using Rust, starting with a core configuration and building up a "flake lattice" incrementally. The goal is to leverage Rust for managing and templating Nix configurations, enabling a more programmatic and verifiable approach to Nix flake development. + +## Overall Goal + +To create a system where Rust programs generate and manage Nix flake definitions, allowing for: +* Dynamic configuration of Nix builds. +* Programmatic generation of Nix expressions. +* A "flake lattice" where each flake adds a feature or dependency, building upon previous ones. + +## Immediate Focus: Generating a Seed Config Flake + +Our immediate goal is to successfully generate a minimal, functional Nix flake directory using Rust. This flake will contain a `flake.nix` generated from a Rust template and will expose a `config.toml` file (generated by `bootstrap-config-builder`) as a Nix package. This will serve as our "seed config" flake. + +## Detailed Plan: Small, Verifiable Steps + +### Phase 1: Ensure `bootstrap-config-builder` can generate `config.toml` + +This phase verifies that our Rust component responsible for generating the core configuration is working correctly in isolation. + +1. **Verify `bootstrap-config-builder` output:** + * **Action:** Navigate to the `bootstrap-config-builder` directory. + * **Command:** `cargo run --bin bootstrap-config-generator -- --output generated_config.toml` + * **Verification:** Confirm that `generated_config.toml` is created in the `bootstrap-config-builder` directory and contains the expected TOML content. + * **Debugging:** If this fails, debug errors within `bootstrap-config-builder/src/` (Rust code) until a valid `config.toml` is produced. + +### Phase 2: Create a New Rust Crate for Generating `flake.nix` from a Template + +This phase involves building the Rust component that will take our generated `config.toml` and embed it into a dynamically created `flake.nix`. + +1. **Create a new Rust crate:** + * **Action:** Create a new Rust project (e.g., `flake-template-generator`) within the main project's `vendor/rust/` directory. + * **Purpose:** This crate will be responsible for: + * Reading the `config.toml` generated in Phase 1. + * Reading a predefined `flake.nix` template. + * Substituting placeholders in the template with values from `config.toml` or other dynamic data. + * Writing the resulting `flake.nix` to a specified output directory. +2. **Define a basic `flake.nix` template:** + * **Action:** Create a minimal `flake.nix` template file (e.g., `flake_template.nix`) within the `flake-template-generator` crate's resources. + * **Content (Example):** + ```nix + { + description = "Dynamically generated config flake"; + + outputs = { self, nixpkgs }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; # Assuming aarch64-linux for now + configTomlContent = builtins.readFile ./config.toml; + in + { + packages.aarch64-linux.default = pkgs.runCommand "generated-config-toml" { } '' + mkdir -p $out + echo "${configTomlContent}" > $out/config.toml + ''; + # Add other outputs as needed, e.g., devShells + }; + } + ``` + * **Placeholders:** The template might include placeholders for system architecture, flake inputs, or other dynamic values that the Rust generator will fill in. For this initial step, we'll keep it simple. + +### Phase 3: Integrate and Test the New Flake Generation + +This phase executes the Rust generator and verifies that the dynamically created flake is valid and functional. + +1. **Run `flake-template-generator`:** + * **Action:** Execute the `flake-template-generator` Rust binary. + * **Command (Example):** `cargo run --bin flake-template-generator -- --config-path ../bootstrap-config-builder/generated_config.toml --output-dir target/generated-flake` + * **Verification:** Confirm that a new directory (e.g., `target/generated-flake`) is created, containing a `flake.nix` and a `config.toml`. +2. **Build the generated flake:** + * **Action:** Navigate to the newly generated flake directory (e.g., `target/generated-flake`). + * **Command:** `nix build .#default` (assuming the template exposes the config as the default package). + * **Verification:** Confirm that the build succeeds and the output (`result`) contains the `config.toml` file. + * **Debugging:** If this fails, debug the generated `flake.nix` and the `flake-template-generator` Rust code. + +### Phase 4: Integrate the Generated Flake into the Root Project + +Once the seed config flake can be reliably generated and built, we will integrate it into the main project's root `flake.nix`. + +1. **Add the generated flake as an input to the root `flake.nix`:** + * **Action:** In the root `flake.nix`, add an input pointing to the dynamically generated flake directory (e.g., `configFlake.url = "./target/generated-flake";`). +2. **Consume the generated config:** + * **Action:** Modify the root `flake.nix` to consume the `config.toml` from `configFlake.packages.${pkgs.system}.default` (or whatever the output is named). + * **Purpose:** This will replace the direct dependency on `bootstrap-config-builder` for the `config.toml` content. +3. **Run `nix build .#default` (root):** + * **Verification:** Confirm that the main project can now build using the dynamically generated config. + +This structured approach ensures that each component is tested and verified before integration, making the debugging process much more manageable. diff --git a/docs/README_intro.md b/docs/README_intro.md new file mode 100644 index 00000000..34459d8b --- /dev/null +++ b/docs/README_intro.md @@ -0,0 +1,10 @@ +# rust-bootstrap-nix + +This repository provides a Nix-based development and build environment for Rust projects, with a focus on integrating `sccache` for accelerated compilation and managing the `x.py` build system. It includes various Nix flakes for environment setup, JSON output processing, and build command evaluation, alongside shell scripts for debugging, development, and testing. + +## Key Features + +* **Reproducible Development Environments:** Utilizes Nix flakes to define consistent Python and Rust development shells. +* **`sccache` Integration:** Accelerates Rust compilation through `sccache` caching. +* **`x.py` Build System Support:** Provides tools and environments for working with the `x.py` build orchestration script. +* **JSON Output Processing:** Includes flakes for capturing and analyzing JSON metadata generated by the build process. diff --git a/docs/README_json_output_generation.md b/docs/README_json_output_generation.md new file mode 100644 index 00000000..4163bd96 --- /dev/null +++ b/docs/README_json_output_generation.md @@ -0,0 +1,43 @@ +## JSON Output Generation + +The flake provides a package that builds the Rust compiler in a "dry run" mode. +In this mode, the build commands are not actually executed, but are captured in JSON files. +This is useful for analyzing the build process and for creating alternative build systems. + +To build the package and generate the JSON files, run the following command from this directory: + +```bash +nix build +``` + +The generated JSON files will be in the `result` directory. + +### Sample JSON Output + +Here is a sample of one of the generated JSON files: + +```json +{ + "command": "/nix/store/lrr9mf5sg6qbas19z1ixjna024zkqws4-rust-default-1.90.0/bin/cargo", + "args": [ + "build", + "--manifest-path", + "/nix/store/qsclyr4nsd25i5p9al261blrki1l9w31-source/standalonex/src/bootstrap/Cargo.toml" + ], + "env": { + "SHELL": "/nix/store/hxmi7d6vbdgbzklm4icfk2y83ncw8la9-bash-5.3p3/bin/bash", + "RUST_BOOTSTRAP_JSON_OUTPUT_DIR": "/nix/store/sc437kd47w1bajlcrdmmgdg0ng57f1l5-xpy-build-output-0.1.0", + "..." + }, + "cwd": "/nix/store/qsclyr4nsd25i5p9al261blrki1l9w31-source/standalonex", + "type": "rust_compiler_invocation" +} +``` + +### Field Explanations + +- `command`: The command to be executed. +- `args`: A list of arguments for the command. +- `env`: A dictionary of environment variables for the command. +- `cwd`: The working directory in which the command should be executed. +- `type`: The type of the invocation. In this case, it's a rust compiler invocation. diff --git a/docs/README_nix_flake_config.md b/docs/README_nix_flake_config.md new file mode 100644 index 00000000..efef3a55 --- /dev/null +++ b/docs/README_nix_flake_config.md @@ -0,0 +1,22 @@ +## 2. `flakes/config/flake.nix` + +**File Path:** `/flakes/config/flake.nix` + +**Description:** This flake is designed to read and process JSON output, specifically `xpy_json_output.json`, which is expected to be generated by the `rust-bootstrap-nix` project. It parses this JSON content and makes it available as a Nix package. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustBootstrapNix`: `github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001` + * **Self-Reference:** This input refers to the main `rust-bootstrap-nix` repository itself, specifically pointing to the `feature/bootstrap-001` branch. This establishes a dependency on the outputs of the main project's flake. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output creates a derivation named `processed-json-output`. + * It reads the `xpy_json_output.json` file from the `rustBootstrapNix.packages.aarch64-linux.default` (which is the `sccache`-enabled Rust compiler package from the root flake). + * The content of `xpy_json_output.json` is parsed as JSON using `builtins.fromJSON`. + * The parsed JSON content is then written to `$out/output.txt` within the derivation. + +**Overall Purpose:** This flake acts as a consumer of the `xpy_json_output.json` file produced by the main `rust-bootstrap-nix` build process. It allows for the structured consumption and further processing of this JSON data within the Nix ecosystem. diff --git a/docs/README_nix_flake_evaluate_rust.md b/docs/README_nix_flake_evaluate_rust.md new file mode 100644 index 00000000..9fe95b6e --- /dev/null +++ b/docs/README_nix_flake_evaluate_rust.md @@ -0,0 +1,27 @@ +## 3. `flakes/evaluate-rust/flake.nix` + +**File Path:** `/flakes/evaluate-rust/flake.nix` + +**Description:** This flake provides a library function `evaluateCommand` designed for recursively evaluating Rust build commands and generating Nix packages. It aims to integrate `naersk` for `cargo build` commands and provides a generic mechanism for other commands. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `naersk`: `github:meta-introspector/naersk?ref=feature/CRQ-016-nixify` + * This input is for `rust2nix` functionality, indicating that this flake intends to use `naersk` to convert Rust projects into Nix derivations. + +**Outputs:** + +* **`lib.evaluateCommand` function:** This is the primary output, a recursive function with the following parameters: + * `commandInfo`: An attribute set containing `command` (the executable, e.g., "cargo", "rustc"), `args` (a list of arguments), and `env` (environment variables). + * `rustSrc`: The source code of the Rust project. + * `currentDepth`: The current recursion depth. + * `maxDepth`: The maximum recursion depth to prevent infinite loops. + + **Function Logic:** + * **Base Case (Recursion Limit):** If `currentDepth` reaches `maxDepth`, it returns a derivation indicating that the recursion limit was reached. + * **`cargo build` Case:** If the command is `cargo` and includes the `build` argument, it uses `naersk.lib.${pkgs.system}.buildPackage` to create a Nix derivation. It passes `cargoBuildFlags` and `env` directly to `naersk`. This is a key integration point for Rust projects. + * **Other Commands Case:** For any other command (e.g., `rustc` directly), it creates a simple `pkgs.runCommand` derivation. It executes the command with its arguments and environment variables, capturing stdout and stderr to `output.txt`. + +**Overall Purpose:** This flake provides a powerful, recursive mechanism to analyze and build Rust projects within Nix. By integrating `naersk`, it can effectively handle `cargo build` commands, transforming them into reproducible Nix derivations. The recursive nature suggests it might be used to trace and build dependencies or stages of a complex Rust build process. diff --git a/docs/README_nix_flake_json_processor.md b/docs/README_nix_flake_json_processor.md new file mode 100644 index 00000000..7e42feff --- /dev/null +++ b/docs/README_nix_flake_json_processor.md @@ -0,0 +1,21 @@ +## 4. `flakes/json-processor/flake.nix` + +**File Path:** `/flakes/json-processor/flake.nix` + +**Description:** This flake defines a Nix package that provides a Python environment with `jq` and `python3` installed. It's intended for processing JSON data, likely in a command-line or scripting context. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. + +**Outputs:** + +* **`packages.aarch64-linux.default` and `packages.x86_64-linux.default`:** + * These outputs define a Nix package for each architecture. + * The package is a `pkgs.mkShell` (which is typically used for development shells, but can also be used to create environments with specific tools). + * **Packages Included:** + * `pkgs.jq`: A lightweight and flexible command-line JSON processor. + * `pkgs.python3`: The Python 3 interpreter. + +**Overall Purpose:** This flake provides a convenient, reproducible environment for working with JSON data using `jq` and Python. It's a utility flake that can be imported by other flakes or used directly to get a shell with these tools. diff --git a/docs/README_nix_flake_json_processor_flake.md b/docs/README_nix_flake_json_processor_flake.md new file mode 100644 index 00000000..d5f99fba --- /dev/null +++ b/docs/README_nix_flake_json_processor_flake.md @@ -0,0 +1,22 @@ +## 5. `flakes/json-processor-flake/flake.nix` + +**File Path:** `/flakes/json-processor-flake/flake.nix` + +**Description:** This flake is very similar to `flakes/config/flake.nix` but specifically targets the `standalonex` flake within the `rust-bootstrap-nix` repository. Its purpose is to read and process the `xpy_json_output.json` generated by the `standalonex` flake. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `standalonex`: `github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=standalonex` + * **Self-Reference:** This input directly references the `standalonex` sub-flake within the `rust-bootstrap-nix` repository, specifically pointing to the `feature/bootstrap-001` branch and the `standalonex` directory. This demonstrates how sub-flakes within the same repository can expose their outputs for consumption by other flakes. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output creates a derivation named `processed-json-output`. + * It reads the `xpy_json_output.json` file from the `standalonex.packages.aarch64-linux.default` (which is the default package output of the `standalonex` flake). + * The content of `xpy_json_output.json` is parsed as JSON using `builtins.fromJSON`. + * The parsed JSON content is then written to `$out/output.txt` within the derivation. + +**Overall Purpose:** This flake serves as a dedicated consumer and processor for the JSON output specifically from the `standalonex` component of the `rust-bootstrap-nix` project. It highlights the modularity of Nix flakes, allowing specific parts of a larger project to expose their outputs for consumption by other flakes. diff --git a/docs/README_nix_flake_minimal_flake.md b/docs/README_nix_flake_minimal_flake.md new file mode 100644 index 00000000..e5ad48d0 --- /dev/null +++ b/docs/README_nix_flake_minimal_flake.md @@ -0,0 +1,23 @@ +## 7. `minimal-flake/flake.nix` + +**File Path:** `/minimal-flake/flake.nix` + +**Description:** This flake provides a very basic Python development environment and a simple "hello world" Python script packaged as a Nix derivation. It serves as a minimal example or a starting point for Python-centric Nix flakes. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. + +**Outputs:** + +* **`devShell`:** + * A development shell named `minimal-python-dev-shell`. + * **Packages Included:** `python3` and `git`. This provides a basic environment for Python development and version control. + +* **`packages..helloPython`:** + * A Nix package named `helloPython` for the `aarch64-linux` system. + * It uses `pkgs.writeScriptBin` to create an executable script. + * The script is a simple Python program that prints "Hello from Nix Python!". + +**Overall Purpose:** This flake demonstrates how to set up a minimal Python development environment and package a simple Python script using Nix. It's likely used for quick testing, as a template, or to illustrate basic Nix flake concepts for Python projects. diff --git a/docs/README_nix_flake_root.md b/docs/README_nix_flake_root.md new file mode 100644 index 00000000..5b243773 --- /dev/null +++ b/docs/README_nix_flake_root.md @@ -0,0 +1,44 @@ +## 1. Root `flake.nix` + +**File Path:** `/flake.nix` + +**Description:** This flake defines a Python and Rust development environment, with a strong emphasis on integrating `sccache` for accelerated Rust compilation. It supports both `aarch64-linux` and `x86_64-linux` systems. The core functionality revolves around providing a customized Rust toolchain that leverages `sccache` during the build process, particularly when running `python x.py build`. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * A custom `nixpkgs` instance, likely providing specific package versions or configurations tailored for the `meta-introspector` ecosystem. +* `rust-overlay`: `github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify` + * A custom Nix overlay for Rust, also sourced from `meta-introspector`, suggesting specialized Rust toolchain management. +* `rustSrcFlake`: `github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2` + * Points to a specific commit of a `rust` repository within `meta-introspector` organization. This appears to be the foundational Rust source that this flake extends and builds upon. + +**Outputs:** + +* **`devShells..default` (for `aarch64-linux` and `x86_64-linux`):** + * Provides a comprehensive development environment. + * **Packages Included:** + * `rustToolchain` (nightly channel, with specific targets configured) + * `python3` + * `python3Packages.pip` + * `git` + * `curl` + * `which` + * **`shellHook`:** Sets `HOME` and `CARGO_HOME` to `$TMPDIR/.cargo` respectively, ensuring a clean and isolated build environment within the shell. + * **`nativeBuildInputs`:** `binutils`, `cmake`, `ninja`, `pkg-config`, `nix`. These are tools required during the build phase. + * **`buildInputs`:** `openssl`, `glibc.out`, `glibc.static`. These are runtime dependencies. + * **Environment Variables:** `RUSTC_ICE` is set to "0", and `LD_LIBRARY_PATH` is configured. + +* **`sccachedRustc` Function:** + * A local function that takes `system`, `pkgs`, and `rustToolchain` as arguments. + * Its primary role is to wrap the `rustSrcFlake`'s default package with `sccache` capabilities. + * **Modifications:** + * Adds `pkgs.sccache` and `pkgs.curl` to `nativeBuildInputs`. + * **`preConfigure`:** Injects environment variables (`RUSTC_WRAPPER`, `SCCACHE_DIR`, `SCCACHE_TEMPDIR`) to enable `sccache` and starts the `sccache` server. + * **`buildPhase`:** Significantly customizes the build process. It creates a `config.toml` file with `vendor = true`, and sets `rustc` and `cargo` paths to the provided `rustToolchain` binaries. It also sets `HOME` and `CARGO_HOME` for the build and executes `python x.py build`. This indicates that `x.py` is a central build orchestration script. + * **`preBuild` and `postBuild`:** Integrates `sccache` statistics reporting (`sccache --zero-stats`, `sccache --show-stats`, `sccache --stop-server`). + +* **`packages..default` (for `aarch64-linux` and `x86_64-linux`):** + * These outputs provide the `sccache`-enabled Rust compiler package, which is the result of applying the `sccachedRustc` function to the respective system's `rustToolchain`. + +**Overall Purpose:** The root `flake.nix` serves as the entry point for setting up a robust, reproducible, and performance-optimized (via `sccache`) development and build environment for a Rust project that likely uses `python x.py build` as its primary build mechanism. It heavily relies on custom `meta-introspector` Nix inputs for its base components. diff --git a/docs/README_nix_flake_standalonex.md b/docs/README_nix_flake_standalonex.md new file mode 100644 index 00000000..200c0118 --- /dev/null +++ b/docs/README_nix_flake_standalonex.md @@ -0,0 +1,42 @@ +## 8. `standalonex/flake.nix` + +**File Path:** `/standalonex/flake.nix` + +**Description:** This flake defines a standalone environment for working with `x.py`, which appears to be a custom build system for Rust projects. It provides a development shell with necessary tools and a package that executes `test_json_output.py` to generate and validate JSON output, likely related to the `x.py` build process. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustSrcFlake`: `github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2` + * The same `rust` source flake used in the root `flake.nix`, providing the `src/stage0` path. +* `rustOverlay`: `github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify` + * The same `rust-overlay` used in the root `flake.nix`. + +**Outputs:** + +* **`devShells.aarch64-linux.default`:** + * A development shell named `standalonex-dev-shell`. + * **Packages Included:** `pkgs.python3`. + * **`shellHook`:** + * Adds the flake's source directory (`${self}/`) to `PATH`, making `x.py` directly executable. + * Sets `RUST_SRC_STAGE0_PATH` to the `src/stage0` directory from `rustSrcFlake`. + * Creates a `config.toml` file with paths to `rustc` and `cargo` from `pkgs.rust-bin.stable.latest.default`. + * Sets `RUST_BOOTSTRAP_CONFIG` to the path of the generated `config.toml`. + * Creates dummy `etc/` files (`rust_analyzer_settings.json`, `rust_analyzer_eglot.el`, `rust_analyzer_helix.toml`) which are likely expected by `x.py` or related tools. + +* **`packages.aarch64-linux.default`:** + * A Nix package named `xpy-build-output`. + * **`src`:** Uses the flake's own source (`self`) as input. + * **`nativeBuildInputs`:** `pkgs.python3` and `pkgs.jq`. + * **`phases`:** Explicitly defines `buildPhase` and `installPhase`. + * **`buildPhase`:** This is the most complex part: + * It creates a writable temporary directory (`$TMPDIR/xpy_work`) and copies the flake's source into it. + * It then copies `config.old.toml` to `config.toml` and uses `sed` to inject the correct `rustc` and `cargo` paths into `config.toml`. + * Sets `RUST_BOOTSTRAP_CONFIG` to the path of the modified `config.toml`. + * Sets `HOME` and `CARGO_HOME` to writable temporary directories. + * Executes `python3 test_json_output.py --output-dir $out` to generate JSON files. + * Validates the generated JSON files using `jq`. + * **`installPhase`:** Is empty, as the output is generated directly in the `buildPhase`. + +**Overall Purpose:** This flake is a self-contained environment for testing and generating output from the `x.py` build system. It meticulously sets up the necessary environment variables, configuration files, and dependencies to run `test_json_output.py`, which in turn uses `x.py` to produce JSON output. This output is then validated and exposed as a Nix package. This flake is crucial for understanding how the `x.py` build system is exercised and how its metadata is captured. diff --git a/docs/README_nix_flake_xpy_json_output_flake.md b/docs/README_nix_flake_xpy_json_output_flake.md new file mode 100644 index 00000000..987a571b --- /dev/null +++ b/docs/README_nix_flake_xpy_json_output_flake.md @@ -0,0 +1,25 @@ +## 6. `flakes/xpy-json-output-flake/flake.nix` + +**File Path:** `/flakes/xpy-json-output-flake/flake.nix` + +**Description:** This flake is specifically designed to execute the `x.py build --json-output` command from the `rustSrc` input and expose the resulting JSON output directory as a Nix package. This is a crucial flake for understanding the build process and its generated metadata. + +**Inputs:** + +* `nixpkgs`: `github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify` + * Standard `nixpkgs` from `meta-introspector`. +* `rustSrc`: `github:meta-introspector/rust?ref=d772ccdfd1905e93362ba045f66dad7e2ccd469b` + * This input points to a specific commit of the `rust` repository within `meta-introspector`. It's marked as `flake = false`, indicating it's treated as a plain source input rather than another Nix flake. This `rustSrc` is where the `x.py` script resides. + +**Outputs:** + +* **`packages.aarch64-linux.default`:** + * This output is a derivation named `xpy-json-output-derivation`. + * It uses `pkgs.runCommandLocal` to execute a local command. + * **`nativeBuildInputs`:** Includes `pkgs.python3` because `x.py` is a Python script. + * **`src`:** The `rustSrc` input is used as the source for this derivation. + * **Build Phase:** + * It creates an output directory `$out`. + * It then executes `python3 $src/x.py build --json-output $out`. This command is responsible for running the `x.py` build script and directing its JSON output to the `$out` directory of this derivation. + +**Overall Purpose:** This flake provides a way to capture and expose the structured JSON output generated by the `x.py` build system of the `rustSrc` project. This output likely contains metadata about the build, such as compilation steps, dependencies, or configuration, which can then be consumed and analyzed by other Nix flakes (like the `json-processor` flakes we've seen). diff --git a/docs/README_nix_flakes_intro.md b/docs/README_nix_flakes_intro.md new file mode 100644 index 00000000..cac74e90 --- /dev/null +++ b/docs/README_nix_flakes_intro.md @@ -0,0 +1 @@ +# Nix Flakes Documentation diff --git a/docs/README_preconditions_intro.md b/docs/README_preconditions_intro.md new file mode 100644 index 00000000..a6758263 --- /dev/null +++ b/docs/README_preconditions_intro.md @@ -0,0 +1,3 @@ +## Preconditions for Nix Flake Build + +The `test_nix_preconditions.sh` script verifies essential environmental setups required for a successful Nix-based build of the Rust bootstrap. Ensuring these preconditions are met helps in maintaining a reproducible and stable build environment. diff --git a/docs/README_preconditions_nix_command.md b/docs/README_preconditions_nix_command.md new file mode 100644 index 00000000..9329ff37 --- /dev/null +++ b/docs/README_preconditions_nix_command.md @@ -0,0 +1,4 @@ +### 1. Nix Command Availability + +* **Check:** Verifies that the `nix` command-line tool is installed and accessible in the system's `PATH`. +* **Importance:** Nix is fundamental to this build system, as it manages dependencies, builds packages, and ensures reproducibility. Without the `nix` command, the build process cannot proceed. diff --git a/docs/README_preconditions_rust_flake.md b/docs/README_preconditions_rust_flake.md new file mode 100644 index 00000000..c8feae8d --- /dev/null +++ b/docs/README_preconditions_rust_flake.md @@ -0,0 +1,4 @@ +### 3. Rust Source Flake (rustSrcFlake) Existence + +* **Check:** Evaluates the Nix store path for the `rustSrcFlake` input (which represents the Rust compiler's source code) as defined in `standalonex/flake.nix`, and verifies that this path exists and contains a known file (`src/ci/channel`). +* **Importance:** The `bootstrap` binary needs to know the location of the Rust compiler's source tree to perform its build tasks. This precondition ensures that the `rustSrcFlake` input is correctly resolved and available, providing the necessary source for the bootstrap process. diff --git a/docs/README_preconditions_rust_sysroot.md b/docs/README_preconditions_rust_sysroot.md new file mode 100644 index 00000000..275cd986 --- /dev/null +++ b/docs/README_preconditions_rust_sysroot.md @@ -0,0 +1,4 @@ +### 2. Rust Toolchain Sysroot Existence + +* **Check:** Evaluates the Nix store path for the `pkgs.rust-bin.stable."1.84.1".default` Rust toolchain (including its source) and confirms that the Rust source directory exists within it. +* **Importance:** The Rust bootstrap process often requires access to the Rust compiler's source code (sysroot) for various build stages and internal operations. This precondition ensures that the necessary source components are available from the Nix-managed Rust toolchain. diff --git a/docs/README_repo_overview.md b/docs/README_repo_overview.md new file mode 100644 index 00000000..255dcf26 --- /dev/null +++ b/docs/README_repo_overview.md @@ -0,0 +1,27 @@ +# Repository Overview: `rust-bootstrap-nix` + +This repository serves as a comprehensive Nix-based environment for developing, building, and testing Rust projects, with a particular focus on integrating `sccache` for build acceleration and leveraging a custom `x.py` build orchestration system. It is designed to provide reproducible build environments across different architectures (`aarch64-linux` and `x86_64-linux`). + +## Core Purpose + +The primary goal of `rust-bootstrap-nix` is to streamline the Rust development workflow within a Nix ecosystem. This involves: + +1. **Reproducible Toolchains:** Providing consistent and isolated Rust compiler and Cargo toolchains via Nix flakes. +2. **Build Acceleration:** Integrating `sccache` to significantly speed up Rust compilation times. +3. **Custom Build Orchestration:** Utilizing a Python-based `x.py` script for managing complex build processes, including dependency handling and build step execution. +4. **Build Metadata Extraction:** Generating and processing structured JSON output from the build process for analysis and further automation. +5. **Modular Flake Structure:** Breaking down the environment and build logic into smaller, interconnected Nix flakes for better organization and reusability. + +## Key Components + +The repository is structured around several key components: + +* **Nix Flakes:** A collection of `flake.nix` files that define development environments, packages, and build logic. These include the root flake, sub-flakes for JSON processing, Rust evaluation, and a standalone `x.py` environment. +* **Shell Scripts:** Various `.sh` scripts for common tasks such as entering development shells, debugging builds, diagnosing environment issues, and updating flakes. +* **Configuration Files:** `config.toml` files that specify build settings, toolchain paths, and vendoring options. +* **`standalonex/` Directory:** A critical component containing the `x.py` build orchestration script, Python utilities (`test_json_output.py`, `wrap_rust.py`), and Rust source code (`src/`). This directory is central to how the Rust project is built and how build metadata is generated. +* **`src/bootstrap/bootstrap.py`:** The core Python script within `standalonex/src/bootstrap/` that implements the detailed logic for the Rust build process, including toolchain management, environment setup, and JSON output generation. + +## How it Works (High-Level) + +The system leverages Nix flakes to define a hermetic build environment. The root `flake.nix` sets up a development shell with Python, Rust, and `sccache`. The `x.py` script (located in `standalonex/`) acts as the primary interface for building the Rust project. During the build, `x.py` (specifically through its `bootstrap` module) can generate JSON output containing detailed information about the compilation steps. Other flakes then consume and process this JSON data, enabling advanced analysis and automation of the Rust build process. diff --git a/docs/README_standalone_xpy_env.md b/docs/README_standalone_xpy_env.md new file mode 100644 index 00000000..530fe47c --- /dev/null +++ b/docs/README_standalone_xpy_env.md @@ -0,0 +1,4 @@ +# Standalone x.py Environment + +This directory contains a standalone version of the `x.py` script from the Rust compiler build system. +It is packaged as a Nix flake that can be built and tested independently. diff --git a/etc/rust_analyzer_eglot.el b/etc/rust_analyzer_eglot.el new file mode 100644 index 00000000..2dfce04e --- /dev/null +++ b/etc/rust_analyzer_eglot.el @@ -0,0 +1 @@ +;; dummy eglot config diff --git a/etc/rust_analyzer_helix.toml b/etc/rust_analyzer_helix.toml new file mode 100644 index 00000000..e982e276 --- /dev/null +++ b/etc/rust_analyzer_helix.toml @@ -0,0 +1 @@ +# dummy helix config diff --git a/etc/rust_analyzer_settings.json b/etc/rust_analyzer_settings.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/etc/rust_analyzer_settings.json @@ -0,0 +1 @@ +{} diff --git a/eval_json.sh b/eval_json.sh new file mode 100755 index 00000000..d99e6bce --- /dev/null +++ b/eval_json.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Hardcoded path to the JSON file in the Nix store +JSON_FILE="/nix/store/hdv212g3rgir248dprwg6bhkz50kkxhb-xpy-build-output-0.1.0/xpy_json_output.json" + +# Check if the JSON file exists +JSON_CONTENT=$(cat "$JSON_FILE") + +# Check if JSON_CONTENT is empty +if [ -z "$JSON_CONTENT" ]; then + echo "Error: JSON content is empty from $JSON_FILE" + exit 1 +fi + +# Use nix eval to parse the JSON string +nix eval --impure --raw --expr " + let + jsonString = builtins.fromJSON \"$JSON_CONTENT\"; + in + jsonString.command +" diff --git a/examples/config-extractor.nix b/examples/config-extractor.nix new file mode 100644 index 00000000..1c58a76a --- /dev/null +++ b/examples/config-extractor.nix @@ -0,0 +1,29 @@ +# config-extractor.nix +{ lib, pkgs, ... }: + +{ + # A function to extract and process configuration from a TOML file. + # configFilePath: Path to the config.toml file. + # extraConfig: Optional attribute set for overriding config values. + extractConfig = { configFilePath, extraConfig ? { } }: + let + # Read the TOML file content + tomlContent = builtins.readFile configFilePath; + + # Parse the TOML content into a Nix attribute set + # This assumes the TOML structure is simple enough for builtins.fromTOML + # For more complex TOML, a custom parser or a Rust tool might be needed. + parsedToml = builtins.fromTOML tomlContent; + + # Merge parsed TOML with extraConfig, with extraConfig taking precedence + finalConfig = lib.recursiveUpdate parsedToml extraConfig; + + # Basic validation (example: check for a 'build' attribute) + validatedConfig = + if builtins.hasAttr "build" finalConfig + then finalConfig + else throw "Configuration missing 'build' section"; + + in + validatedConfig; +} diff --git a/flake-template-generator/Cargo.toml b/flake-template-generator/Cargo.toml new file mode 100644 index 00000000..f6002dca --- /dev/null +++ b/flake-template-generator/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "flake-template-generator" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +toml = "0.8" diff --git a/flake-template-generator/Makefile b/flake-template-generator/Makefile new file mode 100644 index 00000000..10758027 --- /dev/null +++ b/flake-template-generator/Makefile @@ -0,0 +1,22 @@ +DRY_RUN ?= # Set to --dry-run for a dry run, leave empty for actual execution + +.PHONY: all generate-flake + +all: generate-flake + +generate-flake: + @echo "Generating flake directory using flake-template-generator..." + $(eval COMPONENT := solana-rust-1.84.1) + $(eval ARCH := aarch64) + $(eval PHASE := phase0) + $(eval STEP := step1) + $(eval OUTPUT_DIR := flakes/$(COMPONENT)/$(ARCH)/$(PHASE)/$(STEP)) + cargo run --bin flake-template-generator -- \ + --config-path ../bootstrap-config-builder/generated_config.toml \ + --output-dir $(OUTPUT_DIR) \ + --component $(COMPONENT) \ + --arch $(ARCH) \ + --phase $(PHASE) \ + --step $(STEP) \ + $(DRY_RUN) \ + $(ARGS) diff --git a/flake-template-generator/flake_template.nix b/flake-template-generator/flake_template.nix new file mode 100644 index 00000000..fdbea8e3 --- /dev/null +++ b/flake-template-generator/flake_template.nix @@ -0,0 +1,11 @@ +{ + description = "Dynamically generated config flake"; + + outputs = { self }: + let + configTomlContent = builtins.readFile ./config.toml; + in + { + packages.aarch64-linux.default = configTomlContent; # Output the string directly + }; +} diff --git a/flake-template-generator/src/args.rs b/flake-template-generator/src/args.rs new file mode 100644 index 00000000..5e775911 --- /dev/null +++ b/flake-template-generator/src/args.rs @@ -0,0 +1,38 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub struct Args { + /// Path to the generated config.toml + #[arg(long)] + pub config_path: PathBuf, + + /// Output directory for the new flake + #[arg(long)] + pub output_dir: PathBuf, + + /// Component for the branch name: e.g., solana-rust-1.83 + #[arg(long)] + pub component: String, + + /// Architecture for the branch name: e.g., aarch64 + #[arg(long)] + pub arch: String, + + /// Phase for the branch name: e.g., phase0 + #[arg(long)] + pub phase: String, + + /// Step for the branch name: e.g., step1 + #[arg(long)] + pub step: String, + + /// Perform a dry run without executing Git commands + #[arg(long, default_value_t = false)] + pub dry_run: bool, + + /// Show verbose output for Git operations + #[arg(long, default_value_t = false)] + pub verbose: bool, +} diff --git a/flake-template-generator/src/config_parser.rs b/flake-template-generator/src/config_parser.rs new file mode 100644 index 00000000..bf561ea9 --- /dev/null +++ b/flake-template-generator/src/config_parser.rs @@ -0,0 +1,25 @@ +use serde::Deserialize; +use std::fs; +use std::path::PathBuf; + +#[derive(Debug, Deserialize, Default)] +pub struct NixConfig { + #[serde(default)] + pub nixpkgs_path: String, + #[serde(default)] + pub base_branch: String, + // Add other nix-related fields as needed +} + +#[derive(Debug, Deserialize)] +pub struct Config { + #[serde(default)] + pub nix: NixConfig, + // Add other top-level sections as needed +} + +pub fn parse_config(config_path: &PathBuf) -> Result> { + let config_content = fs::read_to_string(config_path)?; + let config: Config = toml::from_str(&config_content)?; + Ok(config) +} diff --git a/flake-template-generator/src/file_writer.rs b/flake-template-generator/src/file_writer.rs new file mode 100644 index 00000000..8ce2a11d --- /dev/null +++ b/flake-template-generator/src/file_writer.rs @@ -0,0 +1,19 @@ +use std::fs; +use std::path::PathBuf; + +pub fn write_flake_and_config( + absolute_output_dir: &PathBuf, + flake_nix_content: &str, + config_content: &str, +) -> Result<(), Box> { + // Write flake.nix to output directory + let output_flake_nix_path = absolute_output_dir.join("flake.nix"); + fs::write(&output_flake_nix_path, flake_nix_content)?; + + // Copy config.toml to output directory + let output_config_toml_path = absolute_output_dir.join("config.toml"); + fs::write(&output_config_toml_path, config_content)?; + + println!("Successfully generated flake in {:?}", absolute_output_dir); + Ok(()) +} diff --git a/flake-template-generator/src/flake_generator.rs b/flake-template-generator/src/flake_generator.rs new file mode 100644 index 00000000..5a10423e --- /dev/null +++ b/flake-template-generator/src/flake_generator.rs @@ -0,0 +1,27 @@ +pub fn generate_flake_nix_content( + nixpkgs_url: &str, + system_arch: &str, +) -> String { + format!( + r#"{{ + description = "Dynamically generated config flake"; + + inputs = {{ + nixpkgs.url = "{}"; + }}; + + outputs = {{ self, nixpkgs }}: + let + pkgs = import nixpkgs {{ system = "{}"; }}; + configTomlContent = builtins.readFile ./config.toml; + in + {{ + packages.{}.default = pkgs.writeText "config.toml" configTomlContent; + }}; +}} +"#, + nixpkgs_url, + system_arch, + system_arch + ) +} diff --git a/flake-template-generator/src/main.rs b/flake-template-generator/src/main.rs new file mode 100644 index 00000000..863267bc --- /dev/null +++ b/flake-template-generator/src/main.rs @@ -0,0 +1,54 @@ +use std::path::PathBuf; +use std::fs; + +mod args; +mod config_parser; +mod flake_generator; +mod file_writer; +mod statix_checker; + +use args::Args; +use clap::Parser; +use config_parser::parse_config; +use flake_generator::generate_flake_nix_content; +use file_writer::write_flake_and_config; +use statix_checker::run_statix_check; + +fn main() -> Result<(), Box> { + let args = Args::parse(); + + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf(); + let absolute_output_dir = repo_root.join(&args.output_dir); + + // Ensure output directory exists + fs::create_dir_all(&absolute_output_dir)?; + + // Parse config.toml + let config = parse_config(&args.config_path)?; + + // Extract nixpkgs_path from config.toml + let nixpkgs_url = if config.nix.nixpkgs_path.is_empty() { + // Fallback to the standard if not specified in config.toml + "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify".to_string() + } else { + config.nix.nixpkgs_path + }; + + // Define the system architecture (can be made dynamic later) + let system_arch = "aarch64-linux"; + + // Generate flake.nix content + let flake_nix_content = generate_flake_nix_content(&nixpkgs_url, &system_arch); + + // Read config.toml content for writing + let config_content = fs::read_to_string(&args.config_path)?; + + // Write flake.nix and config.toml to output directory + write_flake_and_config(&absolute_output_dir, &flake_nix_content, &config_content)?; + + // Run Statix check + let output_flake_nix_path = absolute_output_dir.join("flake.nix"); + run_statix_check(&absolute_output_dir, &output_flake_nix_path)?; + + Ok(()) +} diff --git a/flake-template-generator/src/statix_checker.rs b/flake-template-generator/src/statix_checker.rs new file mode 100644 index 00000000..6c7ac84a --- /dev/null +++ b/flake-template-generator/src/statix_checker.rs @@ -0,0 +1,24 @@ +use std::path::PathBuf; +use std::process::Command; + +pub fn run_statix_check( + absolute_output_dir: &PathBuf, + output_flake_nix_path: &PathBuf, +) -> Result<(), Box> { + println!("Running statix check on generated flake..."); + let statix_output = Command::new("nix-shell") + .arg("-p").arg("statix") + .arg("--run") + .arg(format!("statix check {}", output_flake_nix_path.display())) + .current_dir(absolute_output_dir) // Run statix from the generated flake directory + .output()?; + + if !statix_output.status.success() { + eprintln!("Statix check failed!"); + eprintln!("Stdout: {}", String::from_utf8_lossy(&statix_output.stdout)); + eprintln!("Stderr: {}", String::from_utf8_lossy(&statix_output.stderr)); + return Err("Statix check failed".into()); + } + println!("Statix check passed."); + Ok(()) +} diff --git a/flake.lock b/flake.lock index d568f9f7..8a5863af 100644 --- a/flake.lock +++ b/flake.lock @@ -69,16 +69,16 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1744536153, - "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", - "owner": "NixOS", + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", "repo": "nixpkgs", - "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", "repo": "nixpkgs", "type": "github" } @@ -114,11 +114,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1760582142, - "narHash": "sha256-RSLRjAoS75szOc9fFzRi9/jzPbYsiqPISSLZTloaKtM=", + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", "owner": "meta-introspector", "repo": "rust-overlay", - "rev": "9ea094253b9389ba7dd4f18637f66b5824276d1d", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", "type": "github" }, "original": { @@ -135,16 +135,16 @@ "rust-overlay": "rust-overlay_2" }, "locked": { - "lastModified": 1760638400, - "narHash": "sha256-7C00SAPgAaC9UKlgymfY32VwhmlRPhuhuWDdIBGsWP8=", + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", "owner": "meta-introspector", "repo": "rust", - "rev": "e6c1b92d0abaa3f64032d6662cbcde980c826ff2", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", "type": "github" }, "original": { "owner": "meta-introspector", - "ref": "e6c1b92d0abaa3f64032d6662cbcde980c826ff2", + "ref": "feature/CRQ-016-nixify", "repo": "rust", "type": "github" } diff --git a/flake.nix b/flake.nix index e594001f..de694abc 100644 --- a/flake.nix +++ b/flake.nix @@ -1,146 +1,33 @@ { - description = "Python development environment extending rust-src with sccache"; + description = "Step 1: Generate config.toml"; inputs = { nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; - rustSrcFlake.url = "github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2"; - + rustSrcFlake.url = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify"; }; - outputs = { self, nixpkgs, rust-overlay, rustSrcFlake } : + outputs = { self, nixpkgs, rust-overlay, rustSrcFlake, ... }@inputs: let - pkgs_aarch64 = import nixpkgs { system = "aarch64-linux"; overlays = [ rust-overlay.overlays.default ]; }; - rustToolchain_aarch64 = pkgs_aarch64.rustChannels.nightly.rust.override { targets = [ "aarch64-unknown-linux-gnu" ]; }; - - pkgs_x86_64 = import nixpkgs { system = "x86_64-linux"; overlays = [ rust-overlay.overlays.default ]; }; - rustToolchain_x86_64 = pkgs_x86_64.rustChannels.nightly.rust.override { targets = [ "x86_64-unknown-linux-gnu" ]; }; - - # Define the sccache-enabled rustc package - sccachedRustc = (system: pkgs: rustToolchain: - let - cargo_bin = "${rustToolchain}/bin/cargo"; - rustc_bin = "${rustToolchain}/bin/rustc"; - cargoHome = "$TMPDIR/.cargo"; - compiler_date = "2024-11-28"; - build_triple = if system == "aarch64-linux" then "aarch64-unknown-linux-gnu" else "x86_64-unknown-linux-gnu"; - in - (rustSrcFlake.packages.${system}.default).overrideAttrs (oldAttrs: { - nativeBuildInputs = (oldAttrs.nativeBuildInputs or []) ++ [ pkgs.sccache pkgs.curl ]; - configurePhase = "# Skip the default configure script"; - preConfigure = pkgs.lib.concatStringsSep "\n" [ - (oldAttrs.preConfigure or "") - "export RUSTC_WRAPPER=\"${pkgs.sccache}/bin/sccache\"" - "export SCCACHE_DIR=\"$TMPDIR/sccache\"" - "export SCCACHE_TEMPDIR=\"$TMPDIR/sccache-tmp\"" - "mkdir -p \"$SCCACHE_DIR\"" - "mkdir -p \"$SCCACHE_TEMPDIR\"" - "sccache --stop-server || true" - "sccache --start-server" - "export PATH=\"${pkgs.curl}/bin:$PATH\"" - "export CURL=\"${pkgs.curl}/bin/curl\"" - ]; - buildPhase = pkgs.lib.concatStringsSep "\n" [ - - "echo \"vendor = true\" >> config.toml" - "echo \"rustc = \\\"${rustc_bin}\\\"\" >> config.toml" - "echo \"cargo = \\\"${cargo_bin}\\\"\" >> config.toml" - "echo \"HOME = \\\"$TMPDIR\\\"\" >> config.toml" - "echo \"CARGO_HOME = \\\"$TMPDIR/.cargo\\\"\" >> config.toml" - "mkdir -p \"$TMPDIR/.cargo\"" - "mkdir -p \"build/${build_triple}/stage0\"" - "echo \"${compiler_date}\" > \"build/${build_triple}/stage0/.rustc-stamp\"" - "export HOME=\"$TMPDIR\"" - "export CARGO_HOME=\"$TMPDIR/.cargo\"" - "python x.py build" - ]; - preBuild = (oldAttrs.preBuild or "") + "sccache --zero-stats"; - postBuild = (oldAttrs.postBuild or "") + "sccache --show-stats\nsccache --stop-server"; - }) - ); - + system = "aarch64-linux"; + pkgs = import nixpkgs { + inherit system; + overlays = [ rust-overlay.overlays.default ]; + }; in { - devShells.aarch64-linux.default = pkgs_aarch64.mkShell { - name = "python-rust-fix-dev-shell"; - - packages = [ - rustToolchain_aarch64 - pkgs_aarch64.python3 - pkgs_aarch64.python3Packages.pip - pkgs_aarch64.git - pkgs_aarch64.curl - pkgs_aarch64.which # Add which to the devShell - ]; - - # Set HOME and CARGO_HOME for the devShell - shellHook = '' - export HOME="$TMPDIR" - export CARGO_HOME="$HOME/.cargo" - mkdir -p $CARGO_HOME + packages.aarch64-linux.default = pkgs.stdenv.mkDerivation { + name = "generate-config"; + src = self; + buildInputs = [ pkgs.cargo pkgs.rustc pkgs.cacert pkgs.nix pkgs.pkg-config pkgs.openssl ]; + buildPhase = '' + export CARGO_HOME=$(mktemp -d) + cargo run --bin bootstrap-config-generator -- --project-root . --rust-src-flake-path /nix/store/rhs81k02n3vg452abxl462g2i6xyadyf-source --version 1.84.1 --target aarch64-unknown-linux-gnu --stage 0 ''; - - nativeBuildInputs = [ - pkgs_aarch64.binutils - pkgs_aarch64.cmake - pkgs_aarch64.ninja - pkgs_aarch64.pkg-config - pkgs_aarch64.nix - ]; - - buildInputs = [ - pkgs_aarch64.openssl - pkgs_aarch64.glibc.out - pkgs_aarch64.glibc.static - ]; - - RUSTC_ICE = "0"; - LD_LIBRARY_PATH = "${pkgs_aarch64.lib.makeLibraryPath [ - pkgs_aarch64.stdenv.cc.cc.lib - ]}"; - }; - - devShells.x86_64-linux.default = pkgs_x86_64.mkShell { - name = "python-rust-fix-dev-shell"; - - packages = [ - rustToolchain_x86_64 - pkgs_x86_64.python3 - pkgs_x86_64.python3Packages.pip - pkgs_x86_64.git - pkgs_x86_64.curl - pkgs_x86_64.which # Add which to the devShell - ]; - - # Set HOME and CARGO_HOME for the devShell - shellHook = '' - export HOME="$TMPDIR" - export CARGO_HOME="$HOME/.cargo" - mkdir -p $CARGO_HOME + installPhase = '' + mkdir -p $out + cp config.toml $out/config.toml ''; - - nativeBuildInputs = [ - pkgs_x86_64.binutils - pkgs_x86_64.cmake - pkgs_x86_64.ninja - pkgs_x86_64.pkg-config - pkgs_x86_64.nix - ]; - - buildInputs = [ - pkgs_x86_64.openssl - pkgs_x86_64.glibc.out - pkgs_x86_64.glibc.static - ]; - - RUSTC_ICE = "0"; - LD_LIBRARY_PATH = "${pkgs_x86_64.lib.makeLibraryPath [ - pkgs_x86_64.stdenv.cc.cc.lib - ]}"; }; - - # Define packages.default to be the sccache-enabled rustc package - packages.aarch64-linux.default = sccachedRustc "aarch64-linux" pkgs_aarch64 rustToolchain_aarch64; - packages.x86_64-linux.default = sccachedRustc "x86_64-linux" pkgs_x86_64 rustToolchain_x86_64; }; } diff --git a/flake.old.nix b/flake.old.nix new file mode 100644 index 00000000..bd2e6ede --- /dev/null +++ b/flake.old.nix @@ -0,0 +1,289 @@ +{ + description = "Python development environment extending rust-src with sccache"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + rustSrcFlake.url = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify"; + configuration-nix.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=configuration-nix"; + configuration-nix.inputs.nixpkgs.follows = "nixpkgs"; + standalonex.url = "path:./standalonex"; + configGenerator.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=configuration-nix"; + + + }; + + outputs = { self, nixpkgs, rust-overlay, rustSrcFlake, flake-utils, configuration-nix, standalonex, configGenerator }: + let + lib = nixpkgs.lib; + pkgs_aarch64 = import nixpkgs { system = "aarch64-linux"; overlays = [ rust-overlay.overlays.default ]; }; + rustToolchain_aarch64 = pkgs_aarch64.rust-bin.stable."1.84.1".default; + + pkgs_x86_64 = import nixpkgs { system = "x86_64-linux"; overlays = [ rust-overlay.overlays.default ]; }; + rustToolchain_x86_64 = pkgs_x86_64.rust-bin.stable."1.84.1".default; + + # Define the sccache-enabled rustc package + # sccachedRustc = (system: pkgs: rustToolchain: + # let + # cargo_bin = "${rustToolchain}/bin/cargo"; + # rustc_bin = "${rustToolchain}/bin/rustc"; + # cargoHome = "$TMPDIR/.cargo"; + # compiler_date = "2024-11-28"; + # build_triple = if system == "aarch64-linux" then "aarch64-unknown-linux-gnu" else "x86_64-unknown-linux-gnu"; + # in + # (rustSrcFlake.packages.${system}.default).overrideAttrs (oldAttrs: { + # nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++ [ pkgs.sccache pkgs.curl ]; + # configurePhase = "# Skip the default configure script"; + # preConfigure = pkgs.lib.concatStringsSep "\n" [ + # (oldAttrs.preConfigure or "") + # "export RUSTC_WRAPPER=\"${pkgs.sccache}/bin/sccache\"" + # "export SCCACHE_DIR=\"$TMPDIR/sccache\"" + # "export SCCACHE_TEMPDIR=\"$TMPDIR/sccache-tmp\"" + # "mkdir -p \"$SCCACHE_DIR\"" + # "mkdir -p \"$SCCACHE_TEMPDIR\"" + # "sccache --stop-server || true" + # "sccache --start-server" + # "export PATH=\"${pkgs.curl}/bin:$PATH\"" + # "export CURL=\"${pkgs.curl}/bin/curl\"" + # ]; + # buildPhase = pkgs.lib.concatStringsSep "\n" [ + + + + # "mkdir -p \"$TMPDIR/.cargo\"" + # "mkdir -p \"build/${build_triple}/stage0\"" + # "echo \"${compiler_date}\" > \"build/${build_triple}/stage0/.rustc-stamp\"" + # "export HOME=\"$TMPDIR\"" + # "export CARGO_HOME=\"$TMPDIR/.cargo\"" + # "python x.py build" + # ]; + # preBuild = (oldAttrs.preBuild or "") + "sccache --zero-stats"; + # postBuild = (oldAttrs.postBuild or "") + "sccache --show-stats\nsccache --stop-server"; + # }) + # ); + + # Define packages.default to be the sccache-enabled rustc package + # packages.aarch64-linux.default = sccachedRustc "aarch64-linux" pkgs_aarch64 rustToolchain_aarch64; + # packages.x86_64-linux.default = sccachedRustc "x86_64-linux" pkgs_x86_64 rustToolchain_x86_64; + + # Import the config-extractor + configExtractor = import (self + "/examples/config-extractor.nix") { + inherit lib; + pkgs = pkgs_aarch64; + }; + + # Example usage: Extract config from standalonex/config.toml + parsedConfig = configExtractor.extractConfig { + configFilePath = self + "/standalonex/config.toml"; + extraConfig = { + build = { + patch-binaries-for-nix = false; + }; + }; + }; + + # Helper function to generate config.toml for a given stage + generateConfigTomlForStage = { system, pkgs, rustToolchain, configurationNix, stageNum }: + pkgs.writeText "config-stage-${toString stageNum}.toml" '' + [build] + rustc = "${rustToolchain}/bin/rustc" + cargo = "${rustToolchain}/bin/cargo" + ''; + + # Generate config.toml for multiple stages + configTomlStages_aarch64 = lib.mapAttrs' (stageNum: config: { name = "configStage${stageNum}"; value = config; }) (lib.genAttrs (map toString (lib.range 0 2)) (stageNum: + generateConfigTomlForStage { + system = "aarch64-linux"; + pkgs = pkgs_aarch64; + rustToolchain = rustToolchain_aarch64; # Use the same toolchain for now + configurationNix = configuration-nix; + stageNum = stageNum; + } + )); + + # Generate config.toml for multiple stages + configTomlStages_x86_64 = lib.mapAttrs' (stageNum: config: { name = "configStage${stageNum}"; value = config; }) (lib.genAttrs (map toString (lib.range 0 2)) (stageNum: + generateConfigTomlForStage { + system = "x86_64-linux"; + pkgs = pkgs_x86_64; + rustToolchain = rustToolchain_x86_64; # Use the same toolchain for now + configurationNix = configuration-nix; + stageNum = stageNum; + } + )); + + # Generate config.toml for stage 0 (aarch64-linux) + generatedConfigToml = generateConfigTomlForStage { + system = "aarch64-linux"; + pkgs = pkgs_aarch64; + rustToolchain = rustToolchain_aarch64; + configurationNix = configuration-nix; + stageNum = "0"; + }; + + # Generate config.toml for stage 0 (x86_64-linux) + generatedConfigToml_x86_64 = generateConfigTomlForStage { + system = "x86_64-linux"; + pkgs = pkgs_x86_64; + rustToolchain = rustToolchain_x86_64; + configurationNix = configuration-nix; + stageNum = "0"; + }; + in + { + packages.aarch64-linux = configTomlStages_aarch64 // { + bootstrapConfigBuilder = pkgs_aarch64.stdenv.mkDerivation { + pname = "rust-bootstrap-config-builder"; + version = "0.1.0"; + + # No source needed, as we are just arranging existing outputs + src = null; + + # Depend on the configTomlStages derivations + configStage0 = configTomlStages_aarch64.configStage0; + configStage1 = configTomlStages_aarch64.configStage1; + configStage2 = configTomlStages_aarch64.configStage2; + + installPhase = '' + mkdir -p $out/standalonex/src/bootstrap/stage0 + mkdir -p $out/standalonex/src/bootstrap/stage1 + mkdir -p $out/standalonex/src/bootstrap/stage2 + + ln -s $configStage0 $out/standalonex/src/bootstrap/stage0/config.toml + ln -s $configStage1 $out/standalonex/src/bootstrap/stage1/config.toml + ln -s $configStage2 $out/standalonex/src/bootstrap/stage2/config.toml + ''; + }; + default = standalonex.packages.${pkgs_aarch64.system}.default.overrideAttrs (oldAttrs: { + postPatch = '' + cp ${generatedConfigToml} config.toml + ''; + }); + + packages.x86_64-linux = configTomlStages_x86_64 // { + bootstrapConfigBuilder = pkgs_x86_64.stdenv.mkDerivation { + pname = "rust-bootstrap-config-builder"; + version = "0.1.0"; + + # No source needed, as we are just arranging existing outputs + src = null; + + # Depend on the configTomlStages derivations + configStage0 = configTomlStages_x86_64.configStage0; + configStage1 = configTomlStages_x86_64.configStage1; + configStage2 = configTomlStages_x86_64.configStage2; + + installPhase = '' + mkdir -p $out/standalonex/src/bootstrap/stage0 + mkdir -p $out/standalonex/src/bootstrap/stage1 + mkdir -p $out/standalonex/src/bootstrap/stage2 + + ln -s $configStage0 $out/standalonex/src/bootstrap/stage0/config.toml + ln -s $configStage1 $out/standalonex/src/bootstrap/stage1/config.toml + ln -s $configStage2 $out/standalonex/src/bootstrap/stage2/config.toml + ''; + }; + default = standalonex.packages.${pkgs_x86_64.system}.default.overrideAttrs (oldAttrs: { + postPatch = '' + cp ${generatedConfigToml_x86_64} config.toml + ''; + }); + }; + + devShells.aarch64-linux.default = pkgs_aarch64.mkShell { + name = "python-rust-fix-dev-shell"; + + packages = [ + rustToolchain_aarch64 + pkgs_aarch64.python3 + pkgs_aarch64.python3Packages.pip + pkgs_aarch64.git + pkgs_aarch64.curl + pkgs_aarch64.which # Add which to the devShell + pkgs_aarch64.statix # Add statix to the devShell + pkgs_aarch64.rust-analyzer # Add rust-analyzer to the devShell + ]; + + # Set HOME and CARGO_HOME for the devShell + shellHook = '' + export HOME="$TMPDIR" + export CARGO_HOME="$HOME/.cargo" + mkdir -p $CARGO_HOME + ''; + + nativeBuildInputs = [ + pkgs_aarch64.binutils + pkgs_aarch64.cmake + pkgs_aarch64.ninja + pkgs_aarch64.pkg-config + pkgs_aarch64.nix + ]; + + buildInputs = [ + pkgs_aarch64.openssl + pkgs_aarch64.glibc.out + pkgs_aarch64.glibc.static + ]; + + RUSTC_ICE = "0"; + LD_LIBRARY_PATH = "${pkgs_aarch64.lib.makeLibraryPath [ + pkgs_aarch64.stdenv.cc.cc.lib + ]}"; + }; + + devShells.x86_64-linux.default = pkgs_x86_64.mkShell { + name = "python-rust-fix-dev-shell"; + + packages = [ + rustToolchain_x86_64 + pkgs_x86_64.python3 + pkgs_x86_64.python3Packages.pip + pkgs_x86_64.git + pkgs_x86_64.curl + pkgs_x86_64.which # Add which to the devShell + pkgs_x86_64.statix # Add statix to the devShell + pkgs_x86_64.rust-analyzer # Add rust-analyzer to the devShell + ]; + + # Set HOME and CARGO_HOME for the devShell + shellHook = '' + export HOME="$TMPDIR" + export CARGO_HOME="$HOME/.cargo" + mkdir -p $CARGO_HOME + ''; + + nativeBuildInputs = [ + pkgs_x86_64.binutils + pkgs_x86_64.cmake + pkgs_x86_64.ninja + pkgs_x86_64.pkg-config + pkgs_x86_64.nix + ]; + + buildInputs = [ + pkgs_x86_64.openssl + pkgs_x86_64.glibc.out + pkgs_x86_64.glibc.static + ]; + + RUSTC_ICE = "0"; + LD_LIBRARY_PATH = "${pkgs_x86_64.lib.makeLibraryPath [ + pkgs_x86_64.stdenv.cc.cc.lib + ]}"; + }; + + apps.aarch64-linux.generateConfig = configuration-nix.apps.aarch64-linux.default; + + apps.x86_64-linux.generateConfig = configuration-nix.apps.x86_64-linux.default; + + nixpkgsOutPath = nixpkgs.outPath; + rustOverlayOutPath = rust-overlay.outPath; + rustBootstrapNixOutPath = self.outPath; + configurationNixOutPath = pkgs_aarch64.runCommand "configuration-nix-outpath" { } '' + echo ${configuration-nix.packages.${pkgs_aarch64.system}.default} > $out + ''; + rustSrcFlakeOutPath = rustSrcFlake.outPath; + }; + }; +} + diff --git a/flakes/bootstrap-builder/cc-flake/flake.lock b/flakes/bootstrap-builder/cc-flake/flake.lock new file mode 100644 index 00000000..3af4b302 --- /dev/null +++ b/flakes/bootstrap-builder/cc-flake/flake.lock @@ -0,0 +1,302 @@ +{ + "nodes": { + "allocator-api2": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1760790639, + "narHash": "sha256-tW7QACjn3B7HUN+Xm1jPLdZlpv2WmJgMSZCGAPuw9YM=", + "owner": "meta-introspector", + "repo": "allocator-api2", + "rev": "291a618795755527fc2c8f9d7e678cecf0e238e5", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "allocator-api2", + "type": "github" + } + }, + "cargo2nix": { + "inputs": { + "allocator-api2": "allocator-api2", + "context": "context", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_3", + "rust-overlay": "rust-overlay_2" + }, + "locked": { + "lastModified": 1760808004, + "narHash": "sha256-ByHnaj2K6dSx7BWDP8Txt+O2LA41Gb/mTLUjvxuFD2M=", + "owner": "meta-introspector", + "repo": "cargo2nix", + "rev": "94d556e2ebf2125f3db581a6a5f870a2c85f2bbb", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "cargo2nix", + "type": "github" + } + }, + "context": { + "flake": false, + "locked": { + "dir": "2025/10/10", + "lastModified": 1759506839, + "narHash": "sha256-TOH4TO9/97K0wC8wpv6mXw7eyKpGlZ0oenYfCkCr1J0=", + "owner": "meta-introspector", + "repo": "streamofrandom", + "rev": "8f40e7de433d7c050d5bc2fe47f1f9eef819c886", + "type": "github" + }, + "original": { + "dir": "2025/10/10", + "owner": "meta-introspector", + "ref": "feature/foaf", + "repo": "streamofrandom", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1746162366, + "narHash": "sha256-5SSSZ/oQkwfcAz/o/6TlejlVGqeK08wyREBQ5qFFPhM=", + "owner": "meta-introspector", + "repo": "flake-compat", + "rev": "0f158086a2ecdbb138cd0429410e44994f1b7e4b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "cargo2nix": "cargo2nix", + "nixpkgs": "nixpkgs_4", + "rust-overlay": "rust-overlay_3" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { + "inputs": { + "nixpkgs": [ + "cargo2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1759890791, + "narHash": "sha256-KN1xkrQ4x6u8plgg43ZiYbQmESxeCKKOzALKjqbn4LM=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "74fcbc183aa6685f86008606bb7824bf2f40adbd", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_3": { + "inputs": { + "nixpkgs": "nixpkgs_5" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/bootstrap-builder/cc-flake/flake.nix b/flakes/bootstrap-builder/cc-flake/flake.nix new file mode 100644 index 00000000..1e37f7f2 --- /dev/null +++ b/flakes/bootstrap-builder/cc-flake/flake.nix @@ -0,0 +1,38 @@ +{ + description = "A flake for building the cc crate"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + cargo2nix.url = "github:meta-introspector/cargo2nix?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, rust-overlay, cargo2nix }: + let + pkgs_aarch64 = import nixpkgs { + system = "aarch64-linux"; + overlays = [ rust-overlay.overlays.default ]; + }; + pkgs_x86_64 = import nixpkgs { + system = "x86_64-linux"; + overlays = [ rust-overlay.overlays.default ]; + }; + + bootstrapSrc = ./../../../../standalonex/src/bootstrap; + + generatedRustPackages_aarch64 = cargo2nix.buildRustPackage { + pkgs = pkgs_aarch64; + src = bootstrapSrc; + }; + + generatedRustPackages_x86_64 = cargo2nix.buildRustPackage { + pkgs = pkgs_x86_64; + src = bootstrapSrc; + }; + in + { + packages.aarch64-linux.default = generatedRustPackages_aarch64.bootstrap; + packages.x86_64-linux.default = generatedRustPackages_x86_64.bootstrap; + }; + +} diff --git a/flakes/bootstrap-builder/flake.lock b/flakes/bootstrap-builder/flake.lock new file mode 100644 index 00000000..6fb1d75e --- /dev/null +++ b/flakes/bootstrap-builder/flake.lock @@ -0,0 +1,81 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "rust-src": "rust-src" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-src": { + "flake": false, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "3487cd3843083db70ee30023f19344568ade9c9f", + "repo": "rust", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/bootstrap-builder/flake.nix b/flakes/bootstrap-builder/flake.nix new file mode 100644 index 00000000..e4ebd55c --- /dev/null +++ b/flakes/bootstrap-builder/flake.nix @@ -0,0 +1,56 @@ +{ + description = "A pure Nix flake to build the Rust bootstrap compiler"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + rust-src = { + url = "github:meta-introspector/rust?ref=3487cd3843083db70ee30023f19344568ade9c9f"; + flake = false; + }; + }; + + outputs = { self, nixpkgs, rust-overlay, rust-src }: + let + pkgs = import nixpkgs { + system = "aarch64-linux"; + overlays = [ rust-overlay.overlays.default ]; + }; + + rust_1_84_1_toolchain = pkgs.rust-bin.stable.latest.default; + rust_1_84_1_rustc_path = "${rust_1_84_1_toolchain}/bin/rustc"; + rust_1_84_1_sysroot = pkgs.runCommand "get-sysroot-1-84-1" { } "${rust_1_84_1_rustc_path} --print sysroot > $out"; + rust_1_84_1_libdir = pkgs.runCommand "get-libdir-1-84-1" { } "echo ${rust_1_84_1_sysroot}/lib/rustlib/${pkgs.stdenv.hostPlatform.config}/lib > $out"; + + in + { + packages.aarch64-linux.default = pkgs.rustPlatform.buildRustPackage { + pname = "bootstrap"; + version = "0.1.0"; + + src = rust-src; # Change src to the root of rust-src + + cargoLock.lockFile = "${rust-src}/src/bootstrap/Cargo.lock"; + cargoHash = "sha256-JO1pHLT+BxJrWnydzgu7VO0bR3dRaMlm0XFyL5FqxzI="; + + # The cargo build command needs to be run from the src/bootstrap directory + # So we will add a preBuild phase to change directory + preBuild = '' + cd src/bootstrap + ''; + + checkPhase = '' + # The bootstrap binary is in $out/bin/rustc + # We need to set the environment variables that the shim expects + export RUSTC_STAGE=0 + export RUSTC_SNAPSHOT="${rust_1_84_1_rustc_path}" + export RUSTC_SYSROOT="${rust_1_84_1_sysroot}" + export RUSTC_SNAPSHOT_LIBDIR="${rust_1_84_1_libdir}" + export LD_LIBRARY_PATH="${rust_1_84_1_libdir}" + + # Run the rustc shim and check its version + $out/bin/rustc --version | grep "rustc 1.84.1" + ''; + }; + }; +} diff --git a/flakes/bootstrap-compiler-flake/flake.lock b/flakes/bootstrap-compiler-flake/flake.lock new file mode 100644 index 00000000..23873705 --- /dev/null +++ b/flakes/bootstrap-compiler-flake/flake.lock @@ -0,0 +1,81 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rust-bootstrap-nix": "rust-bootstrap-nix", + "rust-overlay": "rust-overlay" + } + }, + "rust-bootstrap-nix": { + "flake": false, + "locked": { + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/bootstrap-compiler-flake/flake.nix b/flakes/bootstrap-compiler-flake/flake.nix new file mode 100644 index 00000000..301bad69 --- /dev/null +++ b/flakes/bootstrap-compiler-flake/flake.nix @@ -0,0 +1,30 @@ +{ + description = "A flake for building the bootstrap compiler"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + rust-bootstrap-nix = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify"; + flake = false; + }; + }; + + outputs = { self, nixpkgs, rust-overlay, rust-bootstrap-nix }: + let + pkgs = import nixpkgs { + system = "aarch64-linux"; + overlays = [ rust-overlay.overlays.default ]; + }; + in + { + packages.aarch64-linux.default = pkgs.rustPlatform.buildRustPackage { + pname = "bootstrap"; + version = "0.1.0"; + + src = "${rust-bootstrap-nix}/standalonex/src"; + + cargoLock.lockFile = "${rust-bootstrap-nix}/standalonex/src/bootstrap/Cargo.lock"; + }; + }; +} diff --git a/flakes/bootstrap-from-json-flake/flake.lock b/flakes/bootstrap-from-json-flake/flake.lock new file mode 100644 index 00000000..23873705 --- /dev/null +++ b/flakes/bootstrap-from-json-flake/flake.lock @@ -0,0 +1,81 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rust-bootstrap-nix": "rust-bootstrap-nix", + "rust-overlay": "rust-overlay" + } + }, + "rust-bootstrap-nix": { + "flake": false, + "locked": { + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/bootstrap-from-json-flake/flake.nix b/flakes/bootstrap-from-json-flake/flake.nix new file mode 100644 index 00000000..01fa4e31 --- /dev/null +++ b/flakes/bootstrap-from-json-flake/flake.nix @@ -0,0 +1,52 @@ +{ + description = "A flake that builds the bootstrap compiler from JSON data"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + rust-bootstrap-nix = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify"; + flake = false; + }; + }; + + outputs = { self, nixpkgs, rust-overlay, rust-bootstrap-nix }: + let + pkgs = import nixpkgs { + system = "aarch64-linux"; + overlays = [ rust-overlay.overlays.default ]; + }; + + bootstrapBuildPlan = { + command = "${pkgs.rust-bin.stable.latest.default}/bin/cargo"; + args = [ + "build" + "--manifest-path" + "${rust-bootstrap-nix}/standalonex/src/bootstrap/Cargo.toml" + ]; + }; + + in + { + packages.aarch64-linux.default = pkgs.stdenv.mkDerivation { + pname = "bootstrap-from-json"; + version = "0.1.0"; + + src = rust-bootstrap-nix; + + nativeBuildInputs = [ pkgs.rust-bin.stable.latest.default ]; + + buildPhase = '' + export HOME=$TMPDIR + export CARGO_HOME=$TMPDIR/.cargo + mkdir -p $CARGO_HOME + ${bootstrapBuildPlan.command} ${builtins.concatStringsSep " " bootstrapBuildPlan.args} + ''; + + installPhase = '' + mkdir -p $out/bin + cp target/debug/bootstrap $out/bin/ + ''; + }; + }; +} diff --git a/flakes/config-generator/flake.nix b/flakes/config-generator/flake.nix new file mode 100644 index 00000000..038cbda7 --- /dev/null +++ b/flakes/config-generator/flake.nix @@ -0,0 +1,49 @@ +{ + description = "A virtual Nix package to generate rust-bootstrap-nix config.toml files"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rustBootstrapNix.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify"; + configurationNix.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=configuration-nix"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + flake-utils.url = "github:meta-introspector/flake-utils?ref=feature/CRQ-016-nixify"; + rustSrcFlake.url = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, flake-utils, rustBootstrapNix, configurationNix, rust-overlay } @ inputs: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default ]; }; + lib = pkgs.lib; + + # Helper function to get rustToolchain for a specific system + getRustToolchain = system: pkgs: + pkgs.rustChannels.nightly.rust.override { + targets = [ (if system == "aarch64-linux" then "aarch64-unknown-linux-gnu" else "x86_64-unknown-linux-gnu") ]; + }; + + # Replicate the generateConfigTomlForStage logic from the main flake.nix + generateConfigTomlForStage = { system, stageNum, targetTriple, extraConfig ? { } }: + let + rustToolchain = getRustToolchain system pkgs; + in + pkgs.runCommand "config-stage-${toString stageNum}-${targetTriple}.toml" + { + nativeBuildInputs = [ configurationNix.packages.${system}.default ]; # Only configurationNix is needed + } + '' + ${configurationNix.packages.${system}.default}/bin/configuration-nix "${toString stageNum}" "${targetTriple}" + mv config.toml $out + ''; + + configGeneratorScript = pkgs.writeShellScript "config-generator-app" '' + stageNum="$1" + targetTriple="$2" + if [ -z "$stageNum" ] || [ -z "$targetTriple" ]; then + echo "Usage: $0 " + exit 1 + fi + + # Call the configuration-nix executable directly + ${configurationNix.packages.${system}.default}/bin/configuration-nix "$stageNum" "$targetTriple" + ''; diff --git a/flakes/config/flake.lock b/flakes/config/flake.lock new file mode 100644 index 00000000..b2713c4a --- /dev/null +++ b/flakes/config/flake.lock @@ -0,0 +1,839 @@ +{ + "nodes": { + "configGenerator": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_2", + "rust-overlay": "rust-overlay", + "rustSrcFlake": "rustSrcFlake" + }, + "locked": { + "dir": "configuration-nix", + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "dir": "configuration-nix", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "configuration-nix": { + "inputs": { + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "rustBootstrapNix", + "nixpkgs" + ], + "rust-overlay": "rust-overlay_3", + "rustSrcFlake": "rustSrcFlake_2" + }, + "locked": { + "dir": "configuration-nix", + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "dir": "configuration-nix", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_5": { + "inputs": { + "systems": "systems_5" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "id": "flake-utils", + "type": "indirect" + } + }, + "flake-utils_6": { + "inputs": { + "systems": "systems_6" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_7": { + "inputs": { + "systems": "systems_7" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_10": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_11": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_12": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_13": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_14": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_15": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_16": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_6": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_7": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_8": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_9": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rustBootstrapNix": "rustBootstrapNix" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { + "inputs": { + "nixpkgs": "nixpkgs_5" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_3": { + "inputs": { + "nixpkgs": "nixpkgs_6" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_4": { + "inputs": { + "nixpkgs": "nixpkgs_8" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_5": { + "inputs": { + "nixpkgs": "nixpkgs_10" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_6": { + "inputs": { + "nixpkgs": "nixpkgs_12" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_7": { + "inputs": { + "nixpkgs": "nixpkgs_16" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustBootstrapNix": { + "inputs": { + "configGenerator": "configGenerator", + "configuration-nix": "configuration-nix", + "flake-utils": "flake-utils_5", + "nixpkgs": "nixpkgs_9", + "rust-overlay": "rust-overlay_5", + "rustSrcFlake": "rustSrcFlake_3", + "standalonex": "standalonex" + }, + "locked": { + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "rustOverlay": { + "inputs": { + "nixpkgs": "nixpkgs_14" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustSrcFlake": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_4", + "rust-overlay": "rust-overlay_2" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust", + "type": "github" + } + }, + "rustSrcFlake_2": { + "inputs": { + "flake-utils": "flake-utils_4", + "nixpkgs": "nixpkgs_7", + "rust-overlay": "rust-overlay_4" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust", + "type": "github" + } + }, + "rustSrcFlake_3": { + "inputs": { + "flake-utils": "flake-utils_6", + "nixpkgs": "nixpkgs_11", + "rust-overlay": "rust-overlay_6" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust", + "type": "github" + } + }, + "rustSrcFlake_4": { + "inputs": { + "flake-utils": "flake-utils_7", + "nixpkgs": "nixpkgs_15", + "rust-overlay": "rust-overlay_7" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "3487cd3843083db70ee30023f19344568ade9c9f", + "repo": "rust", + "type": "github" + } + }, + "standalonex": { + "inputs": { + "nixpkgs": "nixpkgs_13", + "rustOverlay": "rustOverlay", + "rustSrcFlake": "rustSrcFlake_4" + }, + "locked": { + "dir": "standalonex", + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "dir": "standalonex", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_4": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_5": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_6": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_7": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/config/flake.nix b/flakes/config/flake.nix new file mode 100644 index 00000000..0a4790a1 --- /dev/null +++ b/flakes/config/flake.nix @@ -0,0 +1,23 @@ +{ + description = "Flake to read and process JSON output from rust-bootstrap-nix"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rustBootstrapNix.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, rustBootstrapNix }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; + # Access the xpy_json_output.json from the rustBootstrapNix default package + jsonFile = "${rustBootstrapNix.packages.aarch64-linux.default}/xpy_json_output.json"; + jsonContent = builtins.readFile jsonFile; + parsedJson = builtins.fromJSON jsonContent; + in + { + packages.aarch64-linux.default = pkgs.runCommand "processed-json-output" { } '' + echo "--- Parsed JSON Output ---" > $out/output.txt + echo "${builtins.toJSON parsedJson}" >> $out/output.txt + ''; + }; +} diff --git a/flakes/evaluate-rust/flake.nix b/flakes/evaluate-rust/flake.nix new file mode 100644 index 00000000..ec5cb55b --- /dev/null +++ b/flakes/evaluate-rust/flake.nix @@ -0,0 +1,56 @@ +{ + description = "Flake for evaluating Rust build commands and generating Nix packages recursively."; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + naersk.url = "github:meta-introspector/naersk?ref=feature/CRQ-016-nixify"; # For rust2nix functionality + }; + + outputs = { self, nixpkgs, naersk }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; + + # The core recursive function + evaluateCommand = { commandInfo, rustSrc, currentDepth, maxDepth }: + if currentDepth >= maxDepth then + # Base case: recursion limit reached + [ + (pkgs.runCommand "recursion-limit-reached" { } '' + echo "Recursion limit reached for command: ${commandInfo.command}" > $out/output.txt + '') + ] + else if commandInfo.command == "cargo" && builtins.elem "build" commandInfo.args then + # Case: cargo build command - integrate naersk + [ + (naersk.lib.${pkgs.system}.buildPackage { + pname = "cargo-build-${commandInfo.command}-${builtins.substring 0 8 (builtins.hashString "sha256" (builtins.toJSON commandInfo))}"; + version = "0.1.0"; # Generic version + src = rustSrc; + # Pass cargo arguments directly to naersk + cargoBuildFlags = commandInfo.args; + # Pass environment variables directly to the build + env = commandInfo.env; + }) + ] + else + # Case: other commands (e.g., rustc directly) + [ + (pkgs.runCommand "simple-command-${commandInfo.command}" + { + src = rustSrc; + # Pass the environment variables directly + env = commandInfo.env; + } '' + mkdir -p $out + # Execute the command + ${commandInfo.command} ${builtins.concatStringsSep " " commandInfo.args} > $out/output.txt 2>&1 + '') + ] + ; + in + { + lib = { + inherit evaluateCommand; + }; + }; +} diff --git a/flakes/json-processor-flake/flake.nix b/flakes/json-processor-flake/flake.nix new file mode 100644 index 00000000..5a82b33c --- /dev/null +++ b/flakes/json-processor-flake/flake.nix @@ -0,0 +1,33 @@ +{ + description = "Flake to process JSON output from rust-bootstrap-nix's standalonex flake"; + + inputs = { + #nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + # Reference the standalonex flake within the rust-bootstrap-nix submodule + #standalonex.url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=standalonex"; + }; + + # packages.aarch64-linux.default = pkgs.runCommand "processed-json-output" { } '' + # echo "--- Parsed JSON Output ---" > $out/output.txt + # echo "${builtins.toJSON parsedJson}" >> $out/output.txt + + outputs = + { self + , nixpkgs + #, + #standalonex + }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; + # Access the xpy_json_output.json from the standalonex default package + # jsonFile = "${standalonex.packages.aarch64-linux.default}/xpy_json_output.json"; + # jsonContent = builtins.readFile jsonFile; + # parsedJson = builtins.fromJSON jsonContent; + in + { + # '' + # echo "fixme" >> $out/output.txt + # ''; + }; +} diff --git a/flakes/json-processor/flake.nix b/flakes/json-processor/flake.nix new file mode 100644 index 00000000..2456ca3f --- /dev/null +++ b/flakes/json-processor/flake.nix @@ -0,0 +1,70 @@ +{ + description = "Flake to process JSON output from rust-bootstrap-nix's standalonex flake"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + bootstrapFromJsonFlake = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=flakes/bootstrap-from-json-flake"; + }; + # Reference the main Rust source code + rustSrc = { + url = "github:meta-introspector/rust?ref=feature/CRQ-016-nixify"; + }; + # Reference the evaluate-rust flake + evaluateRustFlake = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=flakes/evaluate-rust"; # Reference the evaluate-rust flake + }; + }; + + outputs = { self, nixpkgs, bootstrapFromJsonFlake, rustSrc, evaluateRustFlake }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; + + # Get the output path from xpyJsonOutputFlake + jsonOutputContent = bootstrapFromJsonFlake.packages.aarch64-linux.default; + + # List all JSON files in the jsonOutput + jsonFiles = builtins.filter (name: builtins.match ".*\\.json" name != null) (builtins.attrNames (builtins.readDir jsonOutputContent)); + + # Function to read and parse a single JSON file + readAndParseJson = filename: + let + jsonContent = builtins.readFile "${jsonOutputContent}/${filename}"; + in + builtins.fromJSON jsonContent; + + # Parse all JSON files + parsedJsons = builtins.map readAndParseJson jsonFiles; + + # Parse all JSON files and evaluate commands + evaluatedPackages = builtins.concatLists ( + builtins.map + (json: evaluateRustFlake.lib.evaluateCommand { + commandInfo = json; + rustSrc = rustSrc; + currentDepth = 0; + maxDepth = 8; + }) + parsedJsons + ); + + in + let + generatedPackages = builtins.listToAttrs ( + builtins.map + (drv: { + name = drv.name; # Assuming the derivation has a 'name' attribute + value = drv; + }) + evaluatedPackages + ); + in + { + packages.aarch64-linux = generatedPackages // { + default = pkgs.symlinkJoin { + name = "all-processed-jsons"; + paths = builtins.attrValues generatedPackages; + }; + }; + }; +} diff --git a/flakes/local-bootstrap-test/flake.nix b/flakes/local-bootstrap-test/flake.nix new file mode 100644 index 00000000..69e7e9e7 --- /dev/null +++ b/flakes/local-bootstrap-test/flake.nix @@ -0,0 +1,84 @@ +{ + description = "Test flake for local rust-bootstrap-nix mirror"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rustOverlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + rustSrc.url = "github:meta-introspector/rust?ref=d772ccdfd1905e93362ba045f66dad7e2ccd469b"; + naersk.url = "github:meta-introspector/naersk?ref=feature/CRQ-016-nixify"; + + # Local mirror references + rustBootstrapNix = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.rust-overlay.follows = "rustOverlay"; + inputs.rustSrcFlake.follows = "rustSrc"; + }; + + rustBootstrapNixConfig = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=flakes/config"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.rustBootstrapNix.follows = "rustBootstrapNix"; + }; + + rustBootstrapNixXpyJsonOutputFlake = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=flakes/xpy-json-output-flake"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.rustSrc.follows = "rustSrc"; + }; + + rustBootstrapNixJsonProcessor = { + url = "git+file:///data/data/com.termux.nix/files/home/git/meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=flakes/json-processor"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.rustSrc.follows = "rustSrc"; + }; + + rustBootstrapNixEvaluateRustFlake = { + url = "git+file:///data/data/com.termux.nix/files/home/git/meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=flakes/evaluate-rust"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.naersk.follows = "naersk"; + }; + + rustBootstrapNixStandalonex = { + url = "git+file:///data/data/com.termux.nix/files/home/git/meta-introspector/rust-bootstrap-nix?ref=feature/bootstrap-001&dir=standalonex"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.rustSrcFlake.follows = "rustSrc"; + inputs.rustOverlay.follows = "rustOverlay"; + }; + }; + + outputs = + { self + , nixpkgs + , rustOverlay + , rustSrc + , naersk + , rustBootstrapNix + , rustBootstrapNixConfig + , rustBootstrapNixXpyJsonOutputFlake + , rustBootstrapNixJsonProcessor + , rustBootstrapNixEvaluateRustFlake + , rustBootstrapNixStandalonex + }: + let + system = "aarch64-linux"; + pkgs = import nixpkgs { inherit system; }; + in + { + devShells.${system}.default = pkgs.mkShell { + name = "local-bootstrap-test-shell"; + packages = [ + # Example: bring in the default package from the main rust-bootstrap-nix flake + rustBootstrapNix.packages.${system}.default + rustBootstrapNixConfig.packages.${system}.default + rustBootstrapNixXpyJsonOutputFlake.packages.${system}.default + rustBootstrapNixJsonProcessor.packages.${system}.default + rustBootstrapNixStandalonex.packages.${system}.default + ]; + shellHook = '' + echo "Welcome to the local-bootstrap-test-shell!" + echo "You can now access packages from the local rust-bootstrap-nix mirror." + ''; + }; + }; +} diff --git a/flakes/solana-rust-1.84.1/aarch64/phase0/step1/config.toml b/flakes/solana-rust-1.84.1/aarch64/phase0/step1/config.toml new file mode 100644 index 00000000..e45c4db3 --- /dev/null +++ b/flakes/solana-rust-1.84.1/aarch64/phase0/step1/config.toml @@ -0,0 +1,56 @@ +# Generated by bootstrap-config-builder +# +# System: +# Project Root: /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix + +[nix] +nixpkgs_path = "" +base_branch = "feature/CRQ-016-nixify" +rust_overlay_path = "" +rust_bootstrap_nix_path = "" +configuration_nix_path = "" +rust_src_flake_path = "/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src" +rust_bootstrap_nix_flake_ref = "" +rust_src_flake_ref = "" +date = "2025-01-30" + +[rust] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.84.1/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.84.1/bin/cargo" +channel = "stable" +download-rustc = false +parallel-compiler = false +llvm-tools = false +debuginfo-level = 0 + +[build] +stage = "" +target = "" +patch-binaries-for-nix = false +vendor = false +build-dir = "" +jobs = 0 + +[env] +HOME = "" +CARGO_HOME = "" + +[install] +prefix = "" +sysconfdir = "" + +[dist] +sign-folder = "" +upload-addr = "" + +[llvm] +download-ci-llvm = false +ninja = false + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "" diff --git a/flakes/solana-rust-1.84.1/aarch64/phase0/step1/flake.nix b/flakes/solana-rust-1.84.1/aarch64/phase0/step1/flake.nix new file mode 100644 index 00000000..2aecb630 --- /dev/null +++ b/flakes/solana-rust-1.84.1/aarch64/phase0/step1/flake.nix @@ -0,0 +1,16 @@ +{ + description = "Dynamically generated config flake"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; }; + configTomlContent = builtins.readFile ./config.toml; + in + { + packages.aarch64-linux.default = pkgs.writeText "config.toml" configTomlContent; + }; +} diff --git a/flakes/use-bootstrap-flake/flake.lock b/flakes/use-bootstrap-flake/flake.lock new file mode 100644 index 00000000..78ffe4ca --- /dev/null +++ b/flakes/use-bootstrap-flake/flake.lock @@ -0,0 +1,230 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "meta-introspector", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", + "repo": "nixpkgs", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rustOverlay": "rustOverlay", + "standalonex": "standalonex" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_5" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustOverlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustOverlay_2": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", + "owner": "meta-introspector", + "repo": "rust-overlay", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-overlay", + "type": "github" + } + }, + "rustSrcFlake": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_4", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", + "owner": "meta-introspector", + "repo": "rust", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", + "type": "github" + }, + "original": { + "owner": "meta-introspector", + "ref": "3487cd3843083db70ee30023f19344568ade9c9f", + "repo": "rust", + "type": "github" + } + }, + "standalonex": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rustOverlay": "rustOverlay_2", + "rustSrcFlake": "rustSrcFlake" + }, + "locked": { + "dir": "standalonex", + "lastModified": 1761264010, + "narHash": "sha256-bk8CGdlbGgWZOAMpP/C8OnrY1/9Qkt5PEOqiiMGwy9o=", + "owner": "meta-introspector", + "repo": "rust-bootstrap-nix", + "rev": "d636fccdf25885a624ec285014b7a639b4052e94", + "type": "github" + }, + "original": { + "dir": "standalonex", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", + "repo": "rust-bootstrap-nix", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/use-bootstrap-flake/flake.nix b/flakes/use-bootstrap-flake/flake.nix new file mode 100644 index 00000000..f45507b6 --- /dev/null +++ b/flakes/use-bootstrap-flake/flake.nix @@ -0,0 +1,66 @@ +{ + description = "A flake for bootstrapping rust"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + rustOverlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + standalonex = { + url = "github:meta-introspector/rust-bootstrap-nix?ref=feature/CRQ-016-nixify&dir=standalonex"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { self, nixpkgs, rustOverlay, standalonex }: + let + pkgs = import nixpkgs { system = "aarch64-linux"; overlays = [ rustOverlay.overlays.default ]; }; + rust_1_84_1_toolchain = pkgs.rust-bin.stable."1.84.1".default; + bootstrap_path = standalonex.packages.aarch64-linux.default; + rust_1_84_1_rustc_path = "${rust_1_84_1_toolchain}/bin/rustc"; + rust_1_84_1_sysroot = pkgs.runCommand "get-sysroot-1-84-1" { } "${rust_1_84_1_rustc_path} --print sysroot > $out"; + rust_1_84_1_libdir = pkgs.runCommand "get-libdir-1-84-1" { } "echo ${rust_1_84_1_sysroot}/lib/rustlib/${pkgs.stdenv.hostPlatform.config}/lib > $out"; + rustSrcPath = standalonex.inputs.rustSrcFlake.outPath; # Correctly get the rust source path + in + { + devShells.aarch64-linux.default = pkgs.mkShell { + packages = [ + rust_1_84_1_toolchain + bootstrap_path + ]; + + shellHook = '' + export PATH=${bootstrap_path}/bin:$PATH + export RUSTC_STAGE=0 # Treat this as stage 0 + export RUSTC_SNAPSHOT=${rust_1_84_1_rustc_path} + export RUSTC_SYSROOT=${rust_1_84_1_sysroot} + export RUSTC_SNAPSHOT_LIBDIR=${rust_1_84_1_libdir} + export LD_LIBRARY_PATH=${rust_1_84_1_libdir} + # export RUST_BACKTRACE=full + #export LD_DEBUG=all + echo "Bootstrap binary is available in your PATH." + ''; + }; + + packages.aarch64-linux = { + run-bootstrap-and-save-output = pkgs.runCommand "run-bootstrap-output" + { + nativeBuildInputs = [ standalonex.packages.aarch64-linux.default ]; + rustSrc = rustSrcPath; # Pass the rustSrcPath as a build input + } '' + mkdir -p $out/share + ${standalonex.packages.aarch64-linux.default}/bin/bootstrap test tidy --src "$rustSrc" > $out/share/bootstrap_output.txt 2>&1 + ''; + default = self.packages.aarch64-linux.run-bootstrap-and-save-output; + + run-cargo-build-and-save-output = pkgs.runCommand "run-cargo-build-output" + { + nativeBuildInputs = [ pkgs.cargo pkgs.rustc pkgs.git ]; # Added pkgs.git + standalonexSrc = standalonex; + } '' + mkdir -p $out/share + cd $standalonexSrc/src/bootstrap + # Capture both stdout and stderr to the file + cargo build --verbose | tee $out/share/cargo_build_output.txt 2>&1 || true # Continue on error to capture output + ''; + }; + }; +} diff --git a/generated_config.toml b/generated_config.toml new file mode 100644 index 00000000..c56e76c4 --- /dev/null +++ b/generated_config.toml @@ -0,0 +1,54 @@ +# Generated by bootstrap-config-builder +# +# System: +# Project Root: /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix + +[nix] +nixpkgs_path = "" +rust_overlay_path = "" +rust_bootstrap_nix_path = "" +configuration_nix_path = "" +rust_src_flake_path = "/data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src" +rust_bootstrap_nix_flake_ref = "" +rust_src_flake_ref = "" + +[rust] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo" +channel = "stable" +download-rustc = false +parallel-compiler = false +llvm-tools = false +debuginfo-level = 0 + +[build] +stage = +target = "" +patch-binaries-for-nix = false +vendor = false +build-dir = "" +jobs = 0 + +[env] +HOME = "" +CARGO_HOME = "" + +[install] +prefix = "" +sysconfdir = "" + +[dist] +sign-folder = "" +upload-addr = "" + +[llvm] +download-ci-llvm = false +ninja = false + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "" diff --git a/lintall.sh b/lintall.sh new file mode 100644 index 00000000..443e7de0 --- /dev/null +++ b/lintall.sh @@ -0,0 +1,6 @@ +for x in `find -name flake.nix | xargs dirname | sort -u`; +do + echo $x; + nix build $x; + +done diff --git a/minimal-flake/flake.lock b/minimal-flake/flake.lock deleted file mode 100644 index f59a43a5..00000000 --- a/minimal-flake/flake.lock +++ /dev/null @@ -1,27 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1757898380, - "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", - "owner": "meta-introspector", - "repo": "nixpkgs", - "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", - "type": "github" - }, - "original": { - "owner": "meta-introspector", - "ref": "feature/CRQ-016-nixify", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/nix-build-scripts/Makefile b/nix-build-scripts/Makefile new file mode 100644 index 00000000..291d99d7 --- /dev/null +++ b/nix-build-scripts/Makefile @@ -0,0 +1,17 @@ +SHELL := /nix/store/hxmi7d6vbdgbzklm4icfk2y83ncw8la9-bash-5.3p3/bin/bash + +.PHONY: build generate-configuration-nix-lock fast-build + +build: generate-configuration-nix-lock + @echo "Updating Nix flake lock file..." + nix flake update + @echo "Now building the main project (standalonex) using the generated config.toml..." + nix build ../standalonex + +fast-build: generate-configuration-nix-lock + @echo "Building generated config.toml (without flake update)..." + nix build ..#generatedConfigToml + @echo "Generated config.toml is at $(shell nix build ..#generatedConfigToml --no-link --print-out-paths)" + @echo "Now building the main project using the generated config.toml..." + @echo "Now building the main project (standalonex) using the generated config.toml..." + nix build ../standalonex diff --git a/nix-build-scripts/standalonex/src/bootstrap/stage0/config.toml b/nix-build-scripts/standalonex/src/bootstrap/stage0/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/nix-build-scripts/standalonex/src/bootstrap/stage0/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/nix-build-scripts/standalonex/src/bootstrap/stage1/config.toml b/nix-build-scripts/standalonex/src/bootstrap/stage1/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/nix-build-scripts/standalonex/src/bootstrap/stage1/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/nix-build-scripts/standalonex/src/bootstrap/stage2/config.toml b/nix-build-scripts/standalonex/src/bootstrap/stage2/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/nix-build-scripts/standalonex/src/bootstrap/stage2/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/push-clones.sh b/push-clones.sh new file mode 100755 index 00000000..b422d43d --- /dev/null +++ b/push-clones.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -euo pipefail + +LOCAL_GIT_ROOT="/data/data/com.termux.nix/files/home/git/" +GITHUB_ORG="meta-introspector" +REPO_NAMES=("rust-overlay" "rust" "naersk") + +echo "Setting up local Git mirrors for meta-introspector repositories..." + +# Define a mapping of repository names to their original paths +declare -A ORIGINAL_REPO_PATHS_MAP +ORIGINAL_REPO_PATHS_MAP["nixpkgs"]="/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/nixpkgs" +ORIGINAL_REPO_PATHS_MAP["rust-overlay"]="/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/nix/rust-overlay" +ORIGINAL_REPO_PATHS_MAP["rust"]="/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src" +ORIGINAL_REPO_PATHS_MAP["naersk"]="/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/nix/naersk" + +for REPO_NAME in "${REPO_NAMES[@]}"; do + LOCAL_REPO_PATH="${LOCAL_GIT_ROOT}${GITHUB_ORG}/${REPO_NAME}" + ORIGINAL_REPO_PATH="${ORIGINAL_REPO_PATHS_MAP[${REPO_NAME}]}" + + echo "--- Processing ${REPO_NAME} ---" + + # Create the organization directory if it doesn't exist + mkdir -p "${LOCAL_GIT_ROOT}${GITHUB_ORG}" + + # Check if ORIGINAL_REPO_PATH exists + if [ ! -d "${ORIGINAL_REPO_PATH}" ]; then + echo "Warning: Original repository for ${REPO_NAME} not found at ${ORIGINAL_REPO_PATH}. Cannot clone or push. Skipping." + continue + fi + + # If the local bare repository doesn't exist, clone it + if [ ! -d "${LOCAL_REPO_PATH}" ]; then + echo "Cloning ${REPO_NAME} from ${ORIGINAL_REPO_PATH} to ${LOCAL_REPO_PATH}..." + git clone --bare "${ORIGINAL_REPO_PATH}" "${LOCAL_REPO_PATH}" + else + echo "Local bare repository for ${REPO_NAME} already exists at ${LOCAL_REPO_PATH}." + fi + + # Ensure the original repository has a remote pointing to the local mirror + ( + cd "${ORIGINAL_REPO_PATH}" || exit + + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + if ! git remote get-url local_mirror > /dev/null 2>&1; then + echo "Adding local_mirror remote to ${ORIGINAL_REPO_PATH}..." + git remote add local_mirror "${LOCAL_REPO_PATH}" + fi + echo "Pushing all branches of ${REPO_NAME} to local mirror..." + git push local_mirror --all + ) + + echo "--- Finished processing ${REPO_NAME} ---" +done + +echo "All local mirror setups and pushes complete." \ No newline at end of file diff --git a/run_bootstrap_build_with_logs.sh b/run_bootstrap_build_with_logs.sh new file mode 100755 index 00000000..b89e2154 --- /dev/null +++ b/run_bootstrap_build_with_logs.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./run_bootstrap_in_shell.sh diff --git a/run_bootstrap_check.sh b/run_bootstrap_check.sh new file mode 100644 index 00000000..b942fe9a --- /dev/null +++ b/run_bootstrap_check.sh @@ -0,0 +1,9 @@ +export RUSTC_STAGE=0 +export RUSTC_SNAPSHOT="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu/bin/rustc" +export RUSTC_SYSROOT="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu" +export RUSTC_SNAPSHOT_LIBDIR="/nix/store/b29wwnvfjfzkf23l2d077nmw5cncaz5s-rustc-1.84.1-aarch64-unknown-linux-gnu/lib" +export LD_LIBRARY_PATH="/nix/store/x9w1w2c9rycrdkp3ynmwjkyk2v40vyb0-get-libdir-1-84-1" +export RUST_BACKTRACE=full +export LD_DEBUG=files + +/nix/store/8f16fw6m01bqhzlbcc8flvj9y3fh6bhw-bootstrap-0.1.0/bin/bootstrap check --src /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix/standalonex diff --git a/run_bootstrap_config_builder.sh b/run_bootstrap_config_builder.sh new file mode 100755 index 00000000..85daa29c --- /dev/null +++ b/run_bootstrap_config_builder.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "Building bootstrapConfigBuilder using Nix..." + +# Determine the current flake's GitHub reference dynamically if possible, +# or use a hardcoded one for now as per user's instruction. +# For this specific case, we'll use the current directory as the flake reference. +FLAKE_REF="." + +nix build "$FLAKE_REF#packages.aarch64-linux.bootstrapConfigBuilder" diff --git a/run_bootstrap_in_shell.sh b/run_bootstrap_in_shell.sh new file mode 100755 index 00000000..efeb507c --- /dev/null +++ b/run_bootstrap_in_shell.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +LOG_FILE="bootstrap_build_output.log" + +echo "Entering Nix develop shell and running bootstrap build..." +unset LD_DEBUG # Unset LD_DEBUG here to reduce verbosity + +# Get the rustSrcPath using nix eval from the use-bootstrap-flake +RUST_SRC_PATH_VAL=$(nix eval --raw --extra-experimental-features "nix-command flakes" \ + "github:meta-introspector/rust?ref=d772ccdfd1905e93362ba045f66dad7e2ccd469b") + +nix develop ./flakes/use-bootstrap-flake#devShells.aarch64-linux.default --command env RUST_SRC_PATH="$RUST_SRC_PATH_VAL" LOG_FILE="$LOG_FILE" bash -c ' + echo "Inside the develop shell." + echo "Running bootstrap build..." + + # The RUST_SRC_PATH and LOG_FILE should now be available as environment variables. + echo "Value of RUST_SRC_PATH: $RUST_SRC_PATH" + + # Execute the bootstrap command with 'check' and the rust source path. + bootstrap check --src "$RUST_SRC_PATH" > "$LOG_FILE" 2>&1 + + echo "Bootstrap build finished. Logs saved to $LOG_FILE" +' + +echo "Script finished." diff --git a/run_bootstrap_test.sh b/run_bootstrap_test.sh new file mode 100755 index 00000000..1f548a78 --- /dev/null +++ b/run_bootstrap_test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euxo pipefail # Added 'x' for debugging + +echo "Running bootstrap test tidy..." + +# Get the path to the standalonex flake +STANDALONEX_FLAKE_REF="github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex" + +# Get the path to the Rust source flake +RUST_SRC_FLAKE_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" --expr ' + let + standalonexFlake = builtins.getFlake "'"$STANDALONEX_FLAKE_REF"'"; + in + standalonexFlake.inputs.rustSrcFlake.outPath +') + +# Get the path to the built bootstrap binary +BOOTSTRAP_DRV_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" "$STANDALONEX_FLAKE_REF#packages.aarch64-linux.default.drv") + +# Get the output path of the built bootstrap package from its derivation +BOOTSTRAP_BINARY_PATH=$(nix-store --query --outputs "$BOOTSTRAP_DRV_PATH") + +echo "BOOTSTRAP_DRV_PATH: $BOOTSTRAP_DRV_PATH" # Debug print +echo "BOOTSTRAP_BINARY_PATH: $BOOTSTRAP_BINARY_PATH" # Debug print + +# Run the bootstrap binary with the correct --src argument +"$BOOTSTRAP_BINARY_PATH/bin/bootstrap" test tidy --src "$RUST_SRC_FLAKE_PATH" \ No newline at end of file diff --git a/run_bootstrap_test_with_use_flake.sh b/run_bootstrap_test_with_use_flake.sh new file mode 100755 index 00000000..5631c6be --- /dev/null +++ b/run_bootstrap_test_with_use_flake.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +echo "Running bootstrap test tidy using flakes/use-bootstrap-flake..." + +USE_BOOTSTRAP_FLAKE_REF="github:meta-introspector/rust-bootstrap-nix?rev=feature/CRQ-016-nixify&dir=flakes/use-bootstrap-flake" + +# Get the path to the Rust source flake (from standalonex's inputs) +RUST_SRC_FLAKE_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" --expr ' + let + standalonexFlake = builtins.getFlake "github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex"; + in + standalonexFlake.inputs.rustSrcFlake.outPath +') +echo RUST_SRC_FLAKE_PATH #RUST_SRC_FLAKE_PATH + +# Define the flake reference for flakes/use-bootstrap-flake +#USE_BOOTSTRAP_FLAKE_REF="github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=flakes/use-bootstrap-flake" + + + +# Run the bootstrap binary within the devShell of flakes/use-bootstrap-flake +nix develop "$USE_BOOTSTRAP_FLAKE_REF#devShells.aarch64-linux.default" --no-write-lock-file --command bash -c "bootstrap test tidy --src \"$RUST_SRC_FLAKE_PATH\"" diff --git a/run_preconditions_test.sh b/run_preconditions_test.sh new file mode 100755 index 00000000..9bf89643 --- /dev/null +++ b/run_preconditions_test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "Running precondition tests using Nix shell..." + +# Determine the current flake's GitHub reference dynamically if possible, +# or use a hardcoded one for now as per user's instruction. +# For this specific case, the user provided the exact reference. +FLAKE_REF="github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex" + +nix shell "$FLAKE_REF#devShells.aarch64-linux.default" -- bash -c "echo 'DevShell loaded successfully'" \ No newline at end of file diff --git a/run_rustc_shim.sh b/run_rustc_shim.sh new file mode 100755 index 00000000..5e86f6c0 --- /dev/null +++ b/run_rustc_shim.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +export RUSTC_STAGE=0 + +RUST_TOOLCHAIN_BASE_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" --expr ' + let + standalonexFlake = builtins.getFlake "github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex"; + pkgs = import standalonexFlake.inputs.nixpkgs { + system = "aarch64-linux"; + overlays = [ standalonexFlake.inputs.rustOverlay.overlays.default ]; + }; + in + pkgs.rust-bin.stable."1.84.1".default +') + +export RUSTC_SNAPSHOT="$RUST_TOOLCHAIN_BASE_PATH/bin/rustc" +export RUSTC_SYSROOT="$RUST_TOOLCHAIN_BASE_PATH" +export RUSTC_SNAPSHOT_LIBDIR="$RUST_TOOLCHAIN_BASE_PATH/lib" +export LD_LIBRARY_PATH="/nix/store/x9w1w2c9rycrdkp3ynmwjkyk2v40vyb0-get-libdir-1-84-1" +export RUST_BACKTRACE=full +# export LD_DEBUG=files + +BOOTSTRAP_RUSTC_PATH=$(nix path-info --json ./standalonex#packages.aarch64-linux.default | jq -r '.[0].path') +"$BOOTSTRAP_RUSTC_PATH/bin/rustc" --version 2>&1 | tee bootstrap_debug_direct.log diff --git a/setup_local_git_mirror.sh b/setup_local_git_mirror.sh new file mode 100755 index 00000000..483ae02f --- /dev/null +++ b/setup_local_git_mirror.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +LOCAL_GIT_ROOT="/data/data/com.termux.nix/files/home/git/" +GITHUB_ORG="meta-introspector" +REPO_NAME="rust-bootstrap-nix" +LOCAL_REPO_PATH="${LOCAL_GIT_ROOT}${GITHUB_ORG}/${REPO_NAME}" +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +echo "Setting up local Git mirror for ${GITHUB_ORG}/${REPO_NAME} at ${LOCAL_REPO_PATH}" + +# Create the organization directory if it doesn't exist +mkdir -p "${LOCAL_GIT_ROOT}${GITHUB_ORG}" + +# If the local repository doesn't exist, clone it +if [ ! -d "${LOCAL_REPO_PATH}" ]; then + echo "Cloning current repository to ${LOCAL_REPO_PATH}..." + git clone --bare . "${LOCAL_REPO_PATH}" +else + echo "Local repository already exists at ${LOCAL_REPO_PATH}. Ensuring it's up-to-date." + # Update the bare repository with the latest from the current working directory + (cd . && git push "${LOCAL_REPO_PATH}" "HEAD:${CURRENT_BRANCH}") +fi + +# Add a local remote to the current repository if it doesn't exist +if ! git remote get-url local_mirror > /dev/null 2>&1; then + echo "Adding local_mirror remote to current repository..." + git remote add local_mirror "${LOCAL_REPO_PATH}" +fi + +# Push the current branch to the local mirror +echo "Pushing current branch (${CURRENT_BRANCH}) to local mirror..." +git push local_mirror "${CURRENT_BRANCH}" + +echo "Local mirror setup and push complete." +echo "You can now reference this repository in your flakes using: git+file://${LOCAL_REPO_PATH}?ref=${CURRENT_BRANCH}" diff --git a/src/ci/channel b/src/ci/channel new file mode 100644 index 00000000..90012116 --- /dev/null +++ b/src/ci/channel @@ -0,0 +1 @@ +dev \ No newline at end of file diff --git a/src/version b/src/version new file mode 100644 index 00000000..d944efab --- /dev/null +++ b/src/version @@ -0,0 +1 @@ +1.84.1 \ No newline at end of file diff --git a/standalonex/config.toml b/standalonex/config.toml index 1456543b..ae4cc6af 100644 --- a/standalonex/config.toml +++ b/standalonex/config.toml @@ -1,2 +1,10 @@ -rustc = "/nix/store/lrr9mf5sg6qbas19z1ixjna024zkqws4-rust-default-1.90.0/bin/rustc" -cargo = "/nix/store/lrr9mf5sg6qbas19z1ixjna024zkqws4-rust-default-1.90.0/bin/cargo" +change-id = 133207 +[build] +patch-binaries-for-nix=true +rust_src_path = "/nix/store/k7wrn478pqvwbzcr7gkbjghcphp62kxd-source" +rustc = "/nix/store/qdz7ccxq8k902nyzc9pb606cp1pam12c-rust-default-1.84.1/bin/rustc" +#/nix/store/qdz7ccxq8k902nyzc9pb606cp1pam12c-rust-default-1.84.1/bin/rustc +cargo = "/nix/store/qdz7ccxq8k902nyzc9pb606cp1pam12c-rust-default-1.84.1/bin/cargo" + +[paths] +cargo = "build/cargo_home" diff --git a/standalonex/devshells.nix b/standalonex/devshells.nix new file mode 100644 index 00000000..e224d206 --- /dev/null +++ b/standalonex/devshells.nix @@ -0,0 +1,38 @@ +{ pkgs +, self +, rustSrcFlake +, +}: + +{ + aarch64-linux.default = pkgs.mkShell { + name = "standalonex-dev-shell"; + + packages = [ + pkgs.python3 + pkgs.rust-bin.stable."1.84.1".default + pkgs.cargo + ]; + + shellHook = '' + # Add the flake's source directory to PATH + export PATH=${self}/:$PATH # self here refers to the flake's source directory in the Nix store + echo "x.py is available in your PATH." + + # Set environment variable for src/stage0 path + export RUST_SRC_STAGE0_PATH=${rustSrcFlake}/src/stage0 + export RUST_SRC_ROOT=${rustSrcFlake} + + # In a Nix environment, it's generally preferred to manage config.toml statically + # or pass tool paths via environment variables to the bootstrap process, + # rather than dynamically generating config.toml in the shellHook. + # For example, RUSTC and CARGO environment variables can be set directly. + + # Create dummy etc/ files for bootstrap compilation + mkdir -p etc + echo "{}" > etc/rust_analyzer_settings.json + echo ";; dummy eglot config" > etc/rust_analyzer_eglot.el + echo "# dummy helix config" > etc/rust_analyzer_helix.toml + ''; + }; +} diff --git a/standalonex/flake.lock b/standalonex/flake.lock index 0429c78d..7c236a5d 100644 --- a/standalonex/flake.lock +++ b/standalonex/flake.lock @@ -69,16 +69,16 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1744536153, - "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", - "owner": "NixOS", + "lastModified": 1757898380, + "narHash": "sha256-1Z0KAfbySsHu/IoZbBJLrPgs3IqQGup0T8J9S1ffAkE=", + "owner": "meta-introspector", "repo": "nixpkgs", - "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "rev": "26833ad1dad83826ef7cc52e0009ca9b7097c79f", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", + "owner": "meta-introspector", + "ref": "feature/CRQ-016-nixify", "repo": "nixpkgs", "type": "github" } @@ -95,11 +95,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1760582142, - "narHash": "sha256-RSLRjAoS75szOc9fFzRi9/jzPbYsiqPISSLZTloaKtM=", + "lastModified": 1760649444, + "narHash": "sha256-K8OdikWA0y9JQxRgd+ZN7KzpS7Zsk/fdv6+WZMcNTOw=", "owner": "meta-introspector", "repo": "rust-overlay", - "rev": "9ea094253b9389ba7dd4f18637f66b5824276d1d", + "rev": "eee7767f08f58eb56822d7e85423098eb3e6dd65", "type": "github" }, "original": { @@ -135,16 +135,16 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1760638400, - "narHash": "sha256-7C00SAPgAaC9UKlgymfY32VwhmlRPhuhuWDdIBGsWP8=", + "lastModified": 1760870238, + "narHash": "sha256-TGYIcYZLXedd4M3LNhqpKHMScQcwo7YOjIWhMmY4tvE=", "owner": "meta-introspector", "repo": "rust", - "rev": "e6c1b92d0abaa3f64032d6662cbcde980c826ff2", + "rev": "3487cd3843083db70ee30023f19344568ade9c9f", "type": "github" }, "original": { "owner": "meta-introspector", - "ref": "e6c1b92d0abaa3f64032d6662cbcde980c826ff2", + "ref": "3487cd3843083db70ee30023f19344568ade9c9f", "repo": "rust", "type": "github" } diff --git a/standalonex/flake.nix b/standalonex/flake.nix index 1981f9c0..6bebb22e 100644 --- a/standalonex/flake.nix +++ b/standalonex/flake.nix @@ -3,16 +3,18 @@ inputs = { nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; - rustSrcFlake.url = "github:meta-introspector/rust?ref=e6c1b92d0abaa3f64032d6662cbcde980c826ff2"; + rustSrcFlake.url = "github:meta-introspector/rust?ref=3487cd3843083db70ee30023f19344568ade9c9f"; rustOverlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; }; - outputs = { self, nixpkgs, rustSrcFlake, rustOverlay }: + outputs = { self, nixpkgs, rustSrcFlake, rustOverlay, ... }: let pkgs = import nixpkgs { system = "aarch64-linux"; overlays = [ rustOverlay.overlays.default ]; }; + rustPlatform = pkgs.rustPlatform; + in { devShells.aarch64-linux.default = pkgs.mkShell { @@ -20,6 +22,8 @@ packages = [ pkgs.python3 + pkgs.rust-bin.stable."1.84.1".default + pkgs.cargo ]; shellHook = '' @@ -29,14 +33,12 @@ # Set environment variable for src/stage0 path export RUST_SRC_STAGE0_PATH=${rustSrcFlake}/src/stage0 + export RUST_SRC_ROOT=${rustSrcFlake} - # Create config.toml with Nix-provided rustc and cargo paths - mkdir -p .cargo - cat > config.toml < $TMPDIR/xpy_json_output.json 2> $TMPDIR/xpy_stderr.log - # Re-enable 'exit on error' - set -e - echo "--- x.py build finished. JSON output captured to $TMPDIR/xpy_json_output.json ---" - - # Read and parse the JSON output within Nix - json_content=$(cat $TMPDIR/xpy_json_output.json) - echo "JSON content read. Now parsing..." - # In a real Nix expression, you would use builtins.fromJSON here. - # For now, we just confirm it's read. - cat $TMPDIR/xpy_json_output.json - - ''; - - installPhase = '' - mkdir -p $out - mv $TMPDIR/xpy_json_output.json $out/xpy_json_output.json - # Print the content of the captured output for debugging - cat $out/xpy_json_output.json - ''; + packages.aarch64-linux = { + default = rustPlatform.buildRustPackage { + pname = "rust-bootstrap-default"; + version = "0.1.0"; + src = pkgs.lib.cleanSource ./src; + cargoLock.lockFile = ./src/Cargo.lock; + rustc = pkgs.rust-bin.stable."1.84.1".default; + doCheck = false; + + bootstrap-main = rustPlatform.buildRustPackage { + pname = "bootstrap-main"; + version = "0.1.0"; + + src = pkgs.lib.cleanSource ./src; + cargoLock.lockFile = ./src/Cargo.lock; + rustc = pkgs.rust-bin.stable."1.84.1".default; + doCheck = false; + cargoBuildFlags = [ "--bin" "bootstrap" ]; + }; + + nix-bootstrap = rustPlatform.buildRustPackage { + pname = "nix-bootstrap"; + version = "0.1.0"; + + src = pkgs.lib.cleanSource ./src; + cargoLock.lockFile = ./src/Cargo.lock; + rustc = pkgs.rust-bin.stable."1.84.1".default; + doCheck = false; + cargoBuildFlags = [ "--bin" "nix_bootstrap" ]; + }; + }; }; }; -} \ No newline at end of file +} + diff --git a/standalonex/generated_config.toml b/standalonex/generated_config.toml new file mode 100644 index 00000000..c56e76c4 --- /dev/null +++ b/standalonex/generated_config.toml @@ -0,0 +1,54 @@ +# Generated by bootstrap-config-builder +# +# System: +# Project Root: /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix + +[nix] +nixpkgs_path = "" +rust_overlay_path = "" +rust_bootstrap_nix_path = "" +configuration_nix_path = "" +rust_src_flake_path = "/data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src" +rust_bootstrap_nix_flake_ref = "" +rust_src_flake_ref = "" + +[rust] +rustc = "/nix/store/yxh9cs2lshqgk6h0kp256yms3w8qwmsz-rustc-wrapper-1.89.0/bin/rustc" +cargo = "/nix/store/ahyjafkgyn6zji9qlvv92z8gxmcmaky4-cargo-1.89.0/bin/cargo" +channel = "stable" +download-rustc = false +parallel-compiler = false +llvm-tools = false +debuginfo-level = 0 + +[build] +stage = +target = "" +patch-binaries-for-nix = false +vendor = false +build-dir = "" +jobs = 0 + +[env] +HOME = "" +CARGO_HOME = "" + +[install] +prefix = "" +sysconfdir = "" + +[dist] +sign-folder = "" +upload-addr = "" + +[llvm] +download-ci-llvm = false +ninja = false + +# Example for target-specific configurations +# [target.{target_triple}] +# cc = "{target_cc}" +# android-ndk = "{target_android_ndk}" + +[change-id] +id = "" diff --git a/standalonex/lib.nix b/standalonex/lib.nix new file mode 100644 index 00000000..ae754df1 --- /dev/null +++ b/standalonex/lib.nix @@ -0,0 +1,21 @@ +{ pkgs +, self +, +}: + +let + parseStage0 = content: + let + lines = builtins.splitString "\n" content; + parsedLines = builtins.filter (line: !(builtins.match "^\\s*#.*" line != null || builtins.match "^\\s*$" line != null)) lines; + keyValues = builtins.map (line: builtins.splitString "=" line) parsedLines; + toAttrSet = builtins.foldl' (acc: kv: acc // { "${builtins.head kv}" = builtins.elemAt kv 1; }) { } keyValues; + in + toAttrSet; + + stage0Attrs = parseStage0 (builtins.readFile (self + "/src/stage0")); +in +{ + parseStage0 = parseStage0; + stage0Attrs = stage0Attrs; +} diff --git a/standalonex/src/Cargo.lock b/standalonex/src/Cargo.lock index efcac4f0..4bdbd4b8 100644 --- a/standalonex/src/Cargo.lock +++ b/standalonex/src/Cargo.lock @@ -13,15 +13,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "block-buffer" @@ -36,12 +36,16 @@ dependencies = [ name = "bootstrap" version = "0.0.0" dependencies = [ - "build_helper", + "bootstrap-config-utils", + "bootstrap-test-utils", "cc", "clap", "clap_complete", "cmake", + "config_core", + "config_macros", "fd-lock", + "globset", "home", "ignore", "junction", @@ -64,22 +68,28 @@ dependencies = [ ] [[package]] -name = "bstr" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +name = "bootstrap-config-utils" +version = "0.1.0" dependencies = [ - "memchr", - "regex-automata", "serde", + "serde_derive", + "stage0_parser_crate", + "toml", ] [[package]] -name = "build_helper" +name = "bootstrap-test-utils" version = "0.1.0" + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ + "memchr", + "regex-automata", "serde", - "serde_derive", ] [[package]] @@ -93,15 +103,15 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "clap" -version = "4.5.18" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623" dependencies = [ "clap_builder", "clap_derive", @@ -109,9 +119,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0" dependencies = [ "anstyle", "clap_lex", @@ -119,18 +129,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.29" +version = "4.5.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e" +checksum = "2348487adcd4631696ced64ccdb40d38ac4d31cae7f2eec8817fcea1b9d1c43c" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -140,9 +150,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" @@ -153,6 +163,24 @@ dependencies = [ "cc", ] +[[package]] +name = "config_core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "config_macros" +version = "0.1.0" +dependencies = [ + "config_core", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -161,18 +189,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -189,9 +217,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -221,42 +249,42 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "fd-lock" -version = "4.0.2" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "filetime" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -264,9 +292,9 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" dependencies = [ "aho-corasick", "bstr", @@ -283,11 +311,11 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -308,31 +336,31 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "junction" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bbdfd737a243da3dfc1f99ee8d6e166480f17ab4ac84d7c34aacd73fc7bd16" +checksum = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8" dependencies = [ "scopeguard", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] name = "libc" -version = "0.2.159" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags", "libc", @@ -341,15 +369,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "log" -version = "0.4.22" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lzma-sys" @@ -364,9 +392,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "ntapi" @@ -379,9 +407,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -398,9 +426,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pretty_assertions" @@ -414,36 +442,36 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -452,28 +480,28 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" -version = "0.38.37" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -492,24 +520,34 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -518,21 +556,22 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -545,11 +584,19 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "stage0_parser_crate" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "syn" -version = "2.0.79" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -571,9 +618,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" dependencies = [ "filetime", "libc", @@ -600,15 +647,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" [[package]] name = "version_check" @@ -644,11 +691,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -664,7 +711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core 0.52.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -674,7 +721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ "windows-core 0.57.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -683,7 +730,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -695,7 +742,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-result", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -720,31 +767,46 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-result" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets", + "windows-link", ] [[package]] @@ -753,14 +815,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -769,56 +848,103 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "xattr" -version = "1.3.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "linux-raw-sys", "rustix", ] diff --git a/standalonex/src/Cargo.nix b/standalonex/src/Cargo.nix new file mode 100644 index 00000000..407951b6 --- /dev/null +++ b/standalonex/src/Cargo.nix @@ -0,0 +1,1423 @@ +# This file was @generated by cargo2nix 0.12.0. +# It is not intended to be manually edited. + +args@{ release ? true +, rootFeatures ? [ + "bootstrap/default" + "build_helper/default" + ] +, rustPackages +, buildRustPackages +, hostPlatform +, hostPlatformCpu ? null +, hostPlatformFeatures ? [ ] +, target ? null +, codegenOpts ? null +, profileOpts ? null +, cargoUnstableFlags ? null +, rustcLinkFlags ? null +, rustcBuildFlags ? null +, mkRustCrate +, rustLib +, lib +, workspaceSrc +, ignoreLockHash +, cargoConfig ? { } +, cargo2nix # Add cargo2nix to arguments +, +}: +let + nixifiedLockHash = "13c465cd179a82a59e780c540ee7d3f1e304b45d4efb2837d96c3d026b88824c"; + workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; + currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); + lockHashIgnored = + if ignoreLockHash + then builtins.trace "Ignoring lock hash" ignoreLockHash + else ignoreLockHash; +in +if !lockHashIgnored && (nixifiedLockHash != currentLockHash) then + throw ("Cargo.nix ${nixifiedLockHash} is out of sync with Cargo.lock ${currentLockHash}") +else + let + inherit (rustLib) fetchCratesIo fetchCrateLocal fetchCrateGit fetchCrateAlternativeRegistry expandFeatures decideProfile; + inherit (cargo2nix) genDrvsByProfile;# Inherit from cargo2nix + cargoConfig' = if cargoConfig != { } then cargoConfig else + if builtins.pathExists ./.cargo/config then lib.importTOML ./.cargo/config else + if builtins.pathExists ./.cargo/config.toml then lib.importTOML ./.cargo/config.toml else { }; + profilesByName = { + dev = builtins.fromTOML "debug = 0\n\n[package.bootstrap]\ndebug = 1\n"; + }; + rootFeatures' = expandFeatures rootFeatures; + overridableMkRustCrate = f: + let + drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ + inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts cargoUnstableFlags rustcLinkFlags rustcBuildFlags; + cargoConfig = cargoConfig'; + } // (f profileName))); + in + { compileMode ? null, profileName ? decideProfile compileMode release }: + let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; }; + in + { + cargo2nixVersion = "0.12.0"; + workspace = { + bootstrap = rustPackages.unknown.bootstrap."0.0.0"; + build_helper = rustPackages.unknown.build_helper."0.1.0"; + }; + "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.1.3" = overridableMkRustCrate (profileName: rec { + name = "aho-corasick"; + version = "1.1.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"; }; + features = builtins.concatLists [ + [ "default" ] + [ "perf-literal" ] + [ "std" ] + ]; + dependencies = { + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.13" = overridableMkRustCrate (profileName: rec { + name = "anstyle"; + version = "1.0.13"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".bitflags."2.9.4" = overridableMkRustCrate (profileName: rec { + name = "bitflags"; + version = "2.9.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.4" = overridableMkRustCrate (profileName: rec { + name = "block-buffer"; + version = "0.10.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"; }; + dependencies = { + generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.9" { inherit profileName; }).out; + }; + }); + + "unknown".bootstrap."0.0.0" = overridableMkRustCrate (profileName: rec { + name = "bootstrap"; + version = "0.0.0"; + registry = "unknown"; + src = fetchCrateLocal workspaceSrc; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/bootstrap-self-test") "bootstrap-self-test") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics") "build-metrics") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "sysinfo") + ]; + dependencies = { + build_helper = (rustPackages."unknown".build_helper."0.1.0" { inherit profileName; }).out; + cc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.1.22" { inherit profileName; }).out; + clap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.5.49" { inherit profileName; }).out; + clap_complete = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_complete."4.5.59" { inherit profileName; }).out; + cmake = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cmake."0.1.48" { inherit profileName; }).out; + fd_lock = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fd-lock."4.0.4" { inherit profileName; }).out; + home = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".home."0.5.11" { inherit profileName; }).out; + ignore = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ignore."0.4.24" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "junction" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".junction."1.3.0" { inherit profileName; }).out; + libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + object = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".object."0.36.7" { inherit profileName; }).out; + opener = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opener."0.5.2" { inherit profileName; }).out; + semver = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".semver."1.0.27" { inherit profileName; }).out; + serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" { inherit profileName; }).out; + serde_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.228" { profileName = "__noProfile"; }).out; + serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.145" { inherit profileName; }).out; + sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.9" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "sysinfo" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sysinfo."0.31.4" { inherit profileName; }).out; + tar = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tar."0.4.44" { inherit profileName; }).out; + termcolor = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".termcolor."1.4.1" { inherit profileName; }).out; + toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.11" { inherit profileName; }).out; + walkdir = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".walkdir."2.5.0" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows."0.52.0" { inherit profileName; }).out; + xz2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xz2."0.1.7" { inherit profileName; }).out; + }; + devDependencies = { + pretty_assertions = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_assertions."1.4.1" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".bstr."1.12.0" = overridableMkRustCrate (profileName: rec { + name = "bstr"; + version = "1.12.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "std" ] + [ "unicode" ] + ]; + dependencies = { + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.4.13" { inherit profileName; }).out; + serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" { inherit profileName; }).out; + }; + }); + + "unknown".build_helper."0.1.0" = overridableMkRustCrate (profileName: rec { + name = "build_helper"; + version = "0.1.0"; + registry = "unknown"; + src = fetchCrateLocal workspaceSrc; + dependencies = { + serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" { inherit profileName; }).out; + serde_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.228" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".cc."1.1.22" = overridableMkRustCrate (profileName: rec { + name = "cc"; + version = "1.1.22"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0"; }; + dependencies = { + shlex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".shlex."1.3.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.4" = overridableMkRustCrate (profileName: rec { + name = "cfg-if"; + version = "1.0.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".clap."4.5.49" = overridableMkRustCrate (profileName: rec { + name = "clap"; + version = "4.5.49"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f"; }; + features = builtins.concatLists [ + [ "derive" ] + [ "error-context" ] + [ "help" ] + [ "std" ] + [ "usage" ] + ]; + dependencies = { + clap_builder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_builder."4.5.49" { inherit profileName; }).out; + clap_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.5.49" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".clap_builder."4.5.49" = overridableMkRustCrate (profileName: rec { + name = "clap_builder"; + version = "4.5.49"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730"; }; + features = builtins.concatLists [ + [ "error-context" ] + [ "help" ] + [ "std" ] + [ "usage" ] + ]; + dependencies = { + anstyle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".anstyle."1.0.13" { inherit profileName; }).out; + clap_lex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.7.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".clap_complete."4.5.59" = overridableMkRustCrate (profileName: rec { + name = "clap_complete"; + version = "4.5.59"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2348487adcd4631696ced64ccdb40d38ac4d31cae7f2eec8817fcea1b9d1c43c"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + clap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."4.5.49" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".clap_derive."4.5.49" = overridableMkRustCrate (profileName: rec { + name = "clap_derive"; + version = "4.5.49"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + heck = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.5.0" { inherit profileName; }).out; + proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" { inherit profileName; }).out; + syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.107" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".clap_lex."0.7.6" = overridableMkRustCrate (profileName: rec { + name = "clap_lex"; + version = "0.7.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".cmake."0.1.48" = overridableMkRustCrate (profileName: rec { + name = "cmake"; + version = "0.1.48"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"; }; + dependencies = { + cc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.1.22" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.7" = overridableMkRustCrate (profileName: rec { + name = "core-foundation-sys"; + version = "0.8.7"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "default") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "link") + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.17" = overridableMkRustCrate (profileName: rec { + name = "cpufeatures"; + version = "0.2.17"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"; }; + dependencies = { + ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.vendor.name == "apple" || hostPlatform.parsed.cpu.name == "loongarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-deque."0.8.6" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-deque"; + version = "0.8.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + crossbeam_epoch = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.18" { inherit profileName; }).out; + crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.21" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.18" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-epoch"; + version = "0.9.18"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "std" ] + ]; + dependencies = { + crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.21" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.21" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-utils"; + version = "0.8.21"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" = overridableMkRustCrate (profileName: rec { + name = "crypto-common"; + version = "0.1.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + generic_array = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.9" { inherit profileName; }).out; + typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.19.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".diff."0.1.13" = overridableMkRustCrate (profileName: rec { + name = "diff"; + version = "0.1.13"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" = overridableMkRustCrate (profileName: rec { + name = "digest"; + version = "0.10.7"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "block-buffer" ] + [ "core-api" ] + [ "default" ] + [ "std" ] + ]; + dependencies = { + block_buffer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".block-buffer."0.10.4" { inherit profileName; }).out; + crypto_common = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".errno."0.3.14" = overridableMkRustCrate (profileName: rec { + name = "errno"; + version = "0.3.14"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "hermit" || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.61.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".fd-lock."4.0.4" = overridableMkRustCrate (profileName: rec { + name = "fd-lock"; + version = "4.0.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"; }; + dependencies = { + cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.4" { inherit profileName; }).out; + ${ if hostPlatform.isUnix then "rustix" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."1.1.2" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.59.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".filetime."0.2.26" = overridableMkRustCrate (profileName: rec { + name = "filetime"; + version = "0.2.26"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"; }; + dependencies = { + cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.4" { inherit profileName; }).out; + ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "redox" then "libredox" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libredox."0.1.10" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.60.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".generic-array."0.14.9" = overridableMkRustCrate (profileName: rec { + name = "generic-array"; + version = "0.14.9"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"; }; + features = builtins.concatLists [ + [ "more_lengths" ] + ]; + dependencies = { + typenum = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".typenum."1.19.0" { inherit profileName; }).out; + }; + buildDependencies = { + version_check = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.5" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".globset."0.4.17" = overridableMkRustCrate (profileName: rec { + name = "globset"; + version = "0.4.17"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "eab69130804d941f8075cfd713bf8848a2c3b3f201a9457a11e6f87e1ab62305"; }; + features = builtins.concatLists [ + [ "default" ] + [ "log" ] + ]; + dependencies = { + aho_corasick = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.1.3" { inherit profileName; }).out; + bstr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bstr."1.12.0" { inherit profileName; }).out; + log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.28" { inherit profileName; }).out; + regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.4.13" { inherit profileName; }).out; + regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.8.8" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".heck."0.5.0" = overridableMkRustCrate (profileName: rec { + name = "heck"; + version = "0.5.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".home."0.5.11" = overridableMkRustCrate (profileName: rec { + name = "home"; + version = "0.5.11"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"; }; + dependencies = { + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.59.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".ignore."0.4.24" = overridableMkRustCrate (profileName: rec { + name = "ignore"; + version = "0.4.24"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "81776e6f9464432afcc28d03e52eb101c93b6f0566f52aef2427663e700f0403"; }; + dependencies = { + crossbeam_deque = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-deque."0.8.6" { inherit profileName; }).out; + globset = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".globset."0.4.17" { inherit profileName; }).out; + log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.28" { inherit profileName; }).out; + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + regex_automata = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.4.13" { inherit profileName; }).out; + same_file = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" { inherit profileName; }).out; + walkdir = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".walkdir."2.5.0" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.11" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.15" = overridableMkRustCrate (profileName: rec { + name = "itoa"; + version = "1.0.15"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".junction."1.3.0" = overridableMkRustCrate (profileName: rec { + name = "junction"; + version = "1.3.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8"; }; + features = builtins.concatLists [ + [ "default" ] + [ "unstable_admin" ] + ]; + dependencies = { + ${ if hostPlatform.isWindows then "scopeguard" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.2.0" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.60.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" = overridableMkRustCrate (profileName: rec { + name = "libc"; + version = "0.2.177"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".libredox."0.1.10" = overridableMkRustCrate (profileName: rec { + name = "libredox"; + version = "0.1.10"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"; }; + features = builtins.concatLists [ + [ "call" ] + [ "default" ] + [ "redox_syscall" ] + [ "std" ] + ]; + dependencies = { + bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.9.4" { inherit profileName; }).out; + libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + syscall = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.5.18" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.11.0" = overridableMkRustCrate (profileName: rec { + name = "linux-raw-sys"; + version = "0.11.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"; }; + features = builtins.concatLists [ + [ "auxvec" ] + [ "elf" ] + [ "errno" ] + [ "general" ] + [ "ioctl" ] + [ "no_std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".log."0.4.28" = overridableMkRustCrate (profileName: rec { + name = "log"; + version = "0.4.28"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".lzma-sys."0.1.20" = overridableMkRustCrate (profileName: rec { + name = "lzma-sys"; + version = "0.1.20"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"; }; + dependencies = { + libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + }; + buildDependencies = { + cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.1.22" { profileName = "__noProfile"; }).out; + pkg_config = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.32" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" = overridableMkRustCrate (profileName: rec { + name = "memchr"; + version = "2.7.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".ntapi."0.4.1" = overridableMkRustCrate (profileName: rec { + name = "ntapi"; + version = "0.4.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "default") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "user") + ]; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".object."0.36.7" = overridableMkRustCrate (profileName: rec { + name = "object"; + version = "0.36.7"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"; }; + features = builtins.concatLists [ + [ "archive" ] + [ "coff" ] + [ "read_core" ] + [ "unaligned" ] + ]; + dependencies = { + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".opener."0.5.2" = overridableMkRustCrate (profileName: rec { + name = "opener"; + version = "0.5.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "293c15678e37254c15bd2f092314abb4e51d7fdde05c2021279c12631b54f005"; }; + dependencies = { + ${ if hostPlatform.parsed.kernel.name == "linux" then "bstr" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bstr."1.12.0" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.32" = overridableMkRustCrate (profileName: rec { + name = "pkg-config"; + version = "0.3.32"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".pretty_assertions."1.4.1" = overridableMkRustCrate (profileName: rec { + name = "pretty_assertions"; + version = "1.4.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + diff = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".diff."0.1.13" { inherit profileName; }).out; + yansi = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".yansi."1.0.1" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" = overridableMkRustCrate (profileName: rec { + name = "proc-macro2"; + version = "1.0.101"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"; }; + features = builtins.concatLists [ + [ "default" ] + [ "proc-macro" ] + ]; + dependencies = { + unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.19" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" = overridableMkRustCrate (profileName: rec { + name = "quote"; + version = "1.0.41"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"; }; + features = builtins.concatLists [ + [ "default" ] + [ "proc-macro" ] + ]; + dependencies = { + proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.5.18" = overridableMkRustCrate (profileName: rec { + name = "redox_syscall"; + version = "0.5.18"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"; }; + features = builtins.concatLists [ + [ "default" ] + [ "userspace" ] + ]; + dependencies = { + bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.9.4" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".regex-automata."0.4.13" = overridableMkRustCrate (profileName: rec { + name = "regex-automata"; + version = "0.4.13"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "dfa-onepass" ] + [ "dfa-search" ] + [ "hybrid" ] + [ "meta" ] + [ "nfa" ] + [ "nfa-backtrack" ] + [ "nfa-pikevm" ] + [ "nfa-thompson" ] + [ "perf" ] + [ "perf-inline" ] + [ "perf-literal" ] + [ "perf-literal-multisubstring" ] + [ "perf-literal-substring" ] + [ "std" ] + [ "syntax" ] + ]; + dependencies = { + aho_corasick = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."1.1.3" { inherit profileName; }).out; + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + regex_syntax = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.8.8" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.8.8" = overridableMkRustCrate (profileName: rec { + name = "regex-syntax"; + version = "0.8.8"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".rustix."1.1.2" = overridableMkRustCrate (profileName: rec { + name = "rustix"; + version = "1.1.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "fs" ] + [ "std" ] + ]; + dependencies = { + bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.9.4" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64) || !hostPlatform.isWindows && !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64)) || hostPlatform.isWindows then "libc_errno" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".errno."0.3.14" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64) || !hostPlatform.isWindows && !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64)) then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64) || hostPlatform.parsed.kernel.name == "linux" && !(hostPlatform.parsed.kernel.name == "linux" && (hostPlatform.parsed.cpu.significantByte.name == "littleEndian" || hostPlatform.parsed.cpu.name == "s390x" || hostPlatform.parsed.cpu.name == "powerpc") && (hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.cpu.bits == 64 || hostPlatform.parsed.cpu.name == "riscv64" || hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.cpu.bits == 64)) then "linux_raw_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".linux-raw-sys."0.11.0" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.61.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.20" = overridableMkRustCrate (profileName: rec { + name = "ryu"; + version = "1.0.20"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" = overridableMkRustCrate (profileName: rec { + name = "same-file"; + version = "1.0.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"; }; + dependencies = { + ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.11" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.2.0" = overridableMkRustCrate (profileName: rec { + name = "scopeguard"; + version = "1.2.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".semver."1.0.27" = overridableMkRustCrate (profileName: rec { + name = "semver"; + version = "1.0.27"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" = overridableMkRustCrate (profileName: rec { + name = "serde"; + version = "1.0.228"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "std" ] + ]; + dependencies = { + serde_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_core."1.0.228" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".serde_core."1.0.228" = overridableMkRustCrate (profileName: rec { + name = "serde_core"; + version = "1.0.228"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "result" ] + [ "std" ] + ]; + dependencies = { + ${ if false then "serde_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.228" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".serde_derive."1.0.228" = overridableMkRustCrate (profileName: rec { + name = "serde_derive"; + version = "1.0.228"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" { inherit profileName; }).out; + syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.107" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.145" = overridableMkRustCrate (profileName: rec { + name = "serde_json"; + version = "1.0.145"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + itoa = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.15" { inherit profileName; }).out; + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + ryu = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ryu."1.0.20" { inherit profileName; }).out; + ${ if false then "serde" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" { inherit profileName; }).out; + serde_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_core."1.0.228" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.9" = overridableMkRustCrate (profileName: rec { + name = "sha2"; + version = "0.10.9"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.4" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "i686" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.17" { inherit profileName; }).out; + digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".shlex."1.3.0" = overridableMkRustCrate (profileName: rec { + name = "shlex"; + version = "1.3.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".syn."2.0.107" = overridableMkRustCrate (profileName: rec { + name = "syn"; + version = "2.0.107"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b"; }; + features = builtins.concatLists [ + [ "clone-impls" ] + [ "default" ] + [ "derive" ] + [ "full" ] + [ "parsing" ] + [ "printing" ] + [ "proc-macro" ] + ]; + dependencies = { + proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" { inherit profileName; }).out; + unicode_ident = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.19" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".sysinfo."0.31.4" = overridableMkRustCrate (profileName: rec { + name = "sysinfo"; + version = "0.31.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "system") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "windows") + ]; + dependencies = { + ${ if (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") && (hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios") then "core_foundation_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".core-foundation-sys."0.8.7" { inherit profileName; }).out; + ${ if (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") && !(hostPlatform.parsed.kernel.name == "unknown" || hostPlatform.parsed.cpu.name == "wasm32") then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "memchr" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.7.6" { inherit profileName; }).out; + ${ if (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") && hostPlatform.isWindows then "ntapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.4.1" { inherit profileName; }).out; + ${ if (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") && hostPlatform.isWindows then "windows" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows."0.57.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".tar."0.4.44" = overridableMkRustCrate (profileName: rec { + name = "tar"; + version = "0.4.44"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"; }; + features = builtins.concatLists [ + [ "default" ] + [ "xattr" ] + ]; + dependencies = { + filetime = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".filetime."0.2.26" { inherit profileName; }).out; + ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if hostPlatform.isUnix then "xattr" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xattr."1.6.1" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".termcolor."1.4.1" = overridableMkRustCrate (profileName: rec { + name = "termcolor"; + version = "1.4.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"; }; + dependencies = { + ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.11" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".toml."0.5.11" = overridableMkRustCrate (profileName: rec { + name = "toml"; + version = "0.5.11"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.228" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".typenum."1.19.0" = overridableMkRustCrate (profileName: rec { + name = "typenum"; + version = "1.19.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".unicode-ident."1.0.19" = overridableMkRustCrate (profileName: rec { + name = "unicode-ident"; + version = "1.0.19"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.5" = overridableMkRustCrate (profileName: rec { + name = "version_check"; + version = "0.9.5"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".walkdir."2.5.0" = overridableMkRustCrate (profileName: rec { + name = "walkdir"; + version = "2.5.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"; }; + dependencies = { + same_file = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".same-file."1.0.6" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "winapi_util" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.11" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" = overridableMkRustCrate (profileName: rec { + name = "winapi"; + version = "0.3.9"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "cfg") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "evntrace") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "in6addr") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "inaddr") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "minwinbase") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "ntsecapi") + [ "shellapi" ] + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "windef") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "winioctl") + ]; + dependencies = { + ${ if hostPlatform.config == "i686-pc-windows-gnu" then "winapi_i686_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" { inherit profileName; }).out; + ${ if hostPlatform.config == "x86_64-pc-windows-gnu" then "winapi_x86_64_pc_windows_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".winapi-i686-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { + name = "winapi-i686-pc-windows-gnu"; + version = "0.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".winapi-util."0.1.11" = overridableMkRustCrate (profileName: rec { + name = "winapi-util"; + version = "0.1.11"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"; }; + dependencies = { + ${ if hostPlatform.isWindows then "windows_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.61.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".winapi-x86_64-pc-windows-gnu."0.4.0" = overridableMkRustCrate (profileName: rec { + name = "winapi-x86_64-pc-windows-gnu"; + version = "0.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows."0.52.0" = overridableMkRustCrate (profileName: rec { + name = "windows"; + version = "0.52.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"; }; + features = builtins.concatLists [ + [ "Win32" ] + [ "Win32_Foundation" ] + [ "Win32_Security" ] + [ "Win32_System" ] + [ "Win32_System_Diagnostics" ] + [ "Win32_System_Diagnostics_Debug" ] + [ "Win32_System_JobObjects" ] + [ "Win32_System_ProcessStatus" ] + [ "Win32_System_Threading" ] + [ "Win32_System_Time" ] + [ "default" ] + ]; + dependencies = { + windows_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-core."0.52.0" { inherit profileName; }).out; + windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows."0.57.0" = overridableMkRustCrate (profileName: rec { + name = "windows"; + version = "0.57.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Wdk") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Wdk_System") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Wdk_System_SystemInformation") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Wdk_System_SystemServices") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Wdk_System_Threading") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_Foundation") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_Security") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_Security_Authorization") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Diagnostics") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Diagnostics_Debug") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Kernel") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Memory") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Performance") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Power") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_ProcessStatus") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Registry") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_RemoteDesktop") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_SystemInformation") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_SystemServices") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_System_Threading") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_UI") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "Win32_UI_Shell") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "default") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "std") + ]; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-core."0.57.0" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-core."0.52.0" = overridableMkRustCrate (profileName: rec { + name = "windows-core"; + version = "0.52.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-core."0.57.0" = overridableMkRustCrate (profileName: rec { + name = "windows-core"; + version = "0.57.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "default") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "std") + ]; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_implement" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-implement."0.57.0" { profileName = "__noProfile"; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_interface" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-interface."0.57.0" { profileName = "__noProfile"; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_result" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-result."0.1.2" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-implement."0.57.0" = overridableMkRustCrate (profileName: rec { + name = "windows-implement"; + version = "0.57.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"; }; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.107" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-interface."0.57.0" = overridableMkRustCrate (profileName: rec { + name = "windows-interface"; + version = "0.57.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"; }; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.101" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.41" { inherit profileName; }).out; + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.107" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-link."0.2.1" = overridableMkRustCrate (profileName: rec { + name = "windows-link"; + version = "0.2.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-result."0.1.2" = overridableMkRustCrate (profileName: rec { + name = "windows-result"; + version = "0.1.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "default") + (lib.optional (rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo") "std") + ]; + dependencies = { + ${ if rootFeatures' ? "bootstrap/build-metrics" || rootFeatures' ? "bootstrap/sysinfo" then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.59.0" = overridableMkRustCrate (profileName: rec { + name = "windows-sys"; + version = "0.59.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"; }; + features = builtins.concatLists [ + [ "Win32" ] + [ "Win32_Foundation" ] + [ "Win32_Storage" ] + [ "Win32_Storage_FileSystem" ] + [ "Win32_System" ] + [ "Win32_System_Com" ] + [ "Win32_System_IO" ] + [ "Win32_UI" ] + [ "Win32_UI_Shell" ] + [ "default" ] + ]; + dependencies = { + windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.60.2" = overridableMkRustCrate (profileName: rec { + name = "windows-sys"; + version = "0.60.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"; }; + features = builtins.concatLists [ + [ "Win32" ] + [ "Win32_Foundation" ] + [ "Win32_Security" ] + [ "Win32_Storage" ] + [ "Win32_Storage_FileSystem" ] + [ "Win32_System" ] + [ "Win32_System_IO" ] + [ "Win32_System_Ioctl" ] + [ "Win32_System_SystemServices" ] + [ "Win32_System_Threading" ] + [ "default" ] + ]; + dependencies = { + windows_targets = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.53.5" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.61.2" = overridableMkRustCrate (profileName: rec { + name = "windows-sys"; + version = "0.61.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"; }; + features = builtins.concatLists [ + [ "Win32" ] + [ "Win32_Foundation" ] + [ "Win32_Networking" ] + [ "Win32_Networking_WinSock" ] + [ "Win32_Storage" ] + [ "Win32_Storage_FileSystem" ] + [ "Win32_System" ] + [ "Win32_System_Console" ] + [ "Win32_System_Diagnostics" ] + [ "Win32_System_Diagnostics_Debug" ] + [ "Win32_System_SystemInformation" ] + [ "default" ] + ]; + dependencies = { + windows_link = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-link."0.2.1" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows-targets"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"; }; + dependencies = { + ${ if hostPlatform.config == "aarch64-pc-windows-gnullvm" then "windows_aarch64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.abi.name == "msvc" then "windows_aarch64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "gnu" then "windows_i686_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.config == "i686-pc-windows-gnullvm" then "windows_i686_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnullvm."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "msvc" then "windows_i686_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.abi.name == "gnu" then "windows_x86_64_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.52.6" { inherit profileName; }).out; + ${ if hostPlatform.config == "x86_64-pc-windows-gnullvm" then "windows_x86_64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.52.6" { inherit profileName; }).out; + ${ if (hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "arm64ec") && hostPlatform.parsed.abi.name == "msvc" then "windows_x86_64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.52.6" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.53.5" = overridableMkRustCrate (profileName: rec { + name = "windows-targets"; + version = "0.53.5"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"; }; + dependencies = { + ${ if false then "windows_link" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-link."0.2.1" { inherit profileName; }).out; + ${ if hostPlatform.config == "aarch64-pc-windows-gnullvm" then "windows_aarch64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.abi.name == "msvc" then "windows_aarch64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "gnu" then "windows_i686_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.config == "i686-pc-windows-gnullvm" then "windows_i686_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnullvm."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "i686" && hostPlatform.parsed.abi.name == "msvc" then "windows_i686_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "x86_64" && hostPlatform.parsed.abi.name == "gnu" then "windows_x86_64_gnu" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.53.1" { inherit profileName; }).out; + ${ if hostPlatform.config == "x86_64-pc-windows-gnullvm" then "windows_x86_64_gnullvm" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.53.1" { inherit profileName; }).out; + ${ if (hostPlatform.parsed.cpu.name == "x86_64" || hostPlatform.parsed.cpu.name == "arm64ec") && hostPlatform.parsed.abi.name == "msvc" then "windows_x86_64_msvc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.53.1" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_aarch64_gnullvm"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_gnullvm."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_aarch64_gnullvm"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_aarch64_msvc"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_aarch64_msvc"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_gnu"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_gnu"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnullvm."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_gnullvm"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnullvm."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_gnullvm"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_msvc"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_msvc"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_gnu"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_gnu"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_gnullvm"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnullvm."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_gnullvm"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.52.6" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_msvc"; + version = "0.52.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.53.1" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_msvc"; + version = "0.53.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".xattr."1.6.1" = overridableMkRustCrate (profileName: rec { + name = "xattr"; + version = "1.6.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"; }; + features = builtins.concatLists [ + [ "default" ] + [ "unsupported" ] + ]; + dependencies = { + ${ if hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "netbsd" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.177" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "hurd" then "rustix" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."1.1.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".xz2."0.1.7" = overridableMkRustCrate (profileName: rec { + name = "xz2"; + version = "0.1.7"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"; }; + dependencies = { + lzma_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lzma-sys."0.1.20" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".yansi."1.0.1" = overridableMkRustCrate (profileName: rec { + name = "yansi"; + version = "1.0.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "std" ] + ]; + }); + + } diff --git a/standalonex/src/Cargo.toml b/standalonex/src/Cargo.toml index 7950f100..ab42733a 100644 --- a/standalonex/src/Cargo.toml +++ b/standalonex/src/Cargo.toml @@ -1,93 +1,5 @@ -[package] -name = "bootstrap" -version = "0.0.0" -edition = "2021" -build = "build.rs" -default-run = "bootstrap" - -[features] -build-metrics = ["sysinfo"] -bootstrap-self-test = [] # enabled in the bootstrap unit tests - -[lib] -path = "src/lib.rs" -doctest = false - -[[bin]] -name = "bootstrap" -path = "src/bin/main.rs" -test = false - -[[bin]] -name = "rustc" -path = "src/bin/rustc.rs" -test = false - -[[bin]] -name = "rustdoc" -path = "src/bin/rustdoc.rs" -test = false - -[[bin]] -name = "sccache-plus-cl" -path = "src/bin/sccache-plus-cl.rs" -test = false - -[dependencies] -# Most of the time updating these dependencies requires modifications to the -# bootstrap codebase(e.g., https://github.com/rust-lang/rust/issues/124565); -# otherwise, some targets will fail. That's why these dependencies are explicitly pinned. -cc = "=1.1.22" -cmake = "=0.1.48" - -build_helper = { path = "../build_helper" } -clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } -clap_complete = "4.4" -fd-lock = "4.0" -home = "0.5" -ignore = "0.4" -libc = "0.2" -object = { version = "0.36.3", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } -opener = "0.5" -semver = "1.0" -serde = "1.0" -# Directly use serde_derive rather than through the derive feature of serde to allow building both -# in parallel and to allow serde_json and toml to start building as soon as serde has been built. -serde_derive = "1.0" -serde_json = "1.0" -sha2 = "0.10" -tar = "0.4" -termcolor = "1.4" -toml = "0.5" -walkdir = "2.4" -xz2 = "0.1" - -# Dependencies needed by the build-metrics feature -sysinfo = { version = "0.31.2", default-features = false, optional = true, features = ["system"] } - -[target.'cfg(windows)'.dependencies.junction] -version = "1.0.0" - -[target.'cfg(windows)'.dependencies.windows] -version = "0.52" -features = [ - "Win32_Foundation", - "Win32_Security", - "Win32_System_Diagnostics_Debug", - "Win32_System_JobObjects", - "Win32_System_ProcessStatus", - "Win32_System_Threading", - "Win32_System_Time", +[workspace] +members = [ + "bootstrap", + "stage0_parser_crate", ] - -[dev-dependencies] -pretty_assertions = "1.4" - -# We care a lot about bootstrap's compile times, so don't include debuginfo for -# dependencies, only bootstrap itself. -[profile.dev] -debug = 0 - -[profile.dev.package] -# Only use debuginfo=1 to further reduce compile times. -bootstrap.debug = 1 diff --git a/standalonex/src/README.md b/standalonex/src/README.md deleted file mode 100644 index f036603e..00000000 --- a/standalonex/src/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# Bootstrapping Rust - -This README is aimed at helping to explain how Rust is bootstrapped, -and some of the technical details of the bootstrap build system. - -Note that this README only covers internal information, not how to use the tool. -Please check [bootstrapping dev guide][bootstrapping-dev-guide] for further information. - -[bootstrapping-dev-guide]: https://rustc-dev-guide.rust-lang.org/building/bootstrapping/intro.html - -## Introduction - -The build system defers most of the complicated logic of managing invocations -of rustc and rustdoc to Cargo itself. However, moving through various stages -and copying artifacts is still necessary for it to do. Each time bootstrap -is invoked, it will iterate through the list of predefined steps and execute -each serially in turn if it matches the paths passed or is a default rule. -For each step, bootstrap relies on the step internally being incremental and -parallel. Note, though, that the `-j` parameter to bootstrap gets forwarded -to appropriate test harnesses and such. - -## Build phases - -Bootstrap build system goes through a few phases to actually build the -compiler. What actually happens when you invoke bootstrap is: - -1. The entry point script (`x` for unix like systems, `x.ps1` for windows systems, - `x.py` cross-platform) is run. This script is responsible for downloading the stage0 - compiler/Cargo binaries, and it then compiles the build system itself (this folder). - Finally, it then invokes the actual `bootstrap` binary build system. -2. In Rust, `bootstrap` will slurp up all configuration, perform a number of - sanity checks (whether compilers exist, for example), and then start building the - stage0 artifacts. -3. The stage0 `cargo`, downloaded earlier, is used to build the standard library - and the compiler, and then these binaries are then copied to the `stage1` - directory. That compiler is then used to generate the stage1 artifacts which - are then copied to the stage2 directory, and then finally, the stage2 - artifacts are generated using that compiler. - -The goal of each stage is to (a) leverage Cargo as much as possible and failing -that (b) leverage Rust as much as possible! - -## Directory Layout - -This build system houses all output under the `build` directory, which looks -like this: - -```sh -# Root folder of all output. Everything is scoped underneath here -build/ - - # Location where the stage0 compiler downloads are all cached. This directory - # only contains the tarballs themselves, as they're extracted elsewhere. - cache/ - 2015-12-19/ - 2016-01-15/ - 2016-01-21/ - ... - - # Output directory for building this build system itself. The stage0 - # cargo/rustc are used to build the build system into this location. - bootstrap/ - debug/ - release/ - - # Output of the dist-related steps like dist-std, dist-rustc, and dist-docs - dist/ - - # Temporary directory used for various input/output as part of various stages - tmp/ - - # Each remaining directory is scoped by the "host" triple of compilation at - # hand. - x86_64-unknown-linux-gnu/ - - # The build artifacts for the `compiler-rt` library for the target that - # this folder is under. The exact layout here will likely depend on the - # platform, and this is also built with CMake, so the build system is - # also likely different. - compiler-rt/ - build/ - - # Output folder for LLVM if it is compiled for this target - llvm/ - - # build folder (e.g. the platform-specific build system). Like with - # compiler-rt, this is compiled with CMake - build/ - - # Installation of LLVM. Note that we run the equivalent of 'make install' - # for LLVM, to setup these folders. - bin/ - lib/ - include/ - share/ - ... - - # Output folder for all documentation of this target. This is what's filled - # in whenever the `doc` step is run. - doc/ - - # Output for all compiletest-based test suites - test/ - ui/ - debuginfo/ - ... - - # Location where the stage0 Cargo and Rust compiler are unpacked. This - # directory is purely an extracted and overlaid tarball of these two (done - # by the bootstrap Python script). In theory, the build system does not - # modify anything under this directory afterwards. - stage0/ - - # These to-build directories are the cargo output directories for builds of - # the standard library, the test system, the compiler, and various tools, - # respectively. Internally, these may also - # have other target directories, which represent artifacts being compiled - # from the host to the specified target. - # - # Essentially, each of these directories is filled in by one `cargo` - # invocation. The build system instruments calling Cargo in the right order - # with the right variables to ensure that these are filled in correctly. - stageN-std/ - stageN-test/ - stageN-rustc/ - stageN-tools/ - - # This is a special case of the above directories, **not** filled in via - # Cargo but rather the build system itself. The stage0 compiler already has - # a set of target libraries for its own host triple (in its own sysroot) - # inside of stage0/. When we run the stage0 compiler to bootstrap more - # things, however, we don't want to use any of these libraries (as those are - # the ones that we're building). So essentially, when the stage1 compiler is - # being compiled (e.g. after libstd has been built), *this* is used as the - # sysroot for the stage0 compiler being run. - # - # Basically, this directory is just a temporary artifact used to configure the - # stage0 compiler to ensure that the libstd that we just built is used to - # compile the stage1 compiler. - stage0-sysroot/lib/ - - # These output directories are intended to be standalone working - # implementations of the compiler (corresponding to each stage). The build - # system will link (using hard links) output from stageN-{std,rustc} into - # each of these directories. - # - # In theory these are working rustc sysroot directories, meaning there is - # no extra build output in these directories. - stage1/ - stage2/ - stage3/ -``` - -## Extending bootstrap - -When you use bootstrap, you'll call it through the entry point script -(`x`, `x.ps1`, or `x.py`). However, most of the code lives in `src/bootstrap`. -`bootstrap` has a difficult problem: it is written in Rust, but yet it is run -before the Rust compiler is built! To work around this, there are two components -of bootstrap: the main one written in rust, and `bootstrap.py`. `bootstrap.py` -is what gets run by entry point script. It takes care of downloading the `stage0` -compiler, which will then build the bootstrap binary written in Rust. - -Because there are two separate codebases behind `x.py`, they need to -be kept in sync. In particular, both `bootstrap.py` and the bootstrap binary -parse `config.toml` and read the same command line arguments. `bootstrap.py` -keeps these in sync by setting various environment variables, and the -programs sometimes have to add arguments that are explicitly ignored, to be -read by the other. - -Some general areas that you may be interested in modifying are: - -* Adding a new build tool? Take a look at `bootstrap/src/core/build_steps/tool.rs` - for examples of other tools. -* Adding a new compiler crate? Look no further! Adding crates can be done by - adding a new directory with `Cargo.toml`, followed by configuring all - `Cargo.toml` files accordingly. -* Adding a new dependency from crates.io? This should just work inside the - compiler artifacts stage (everything other than libtest and libstd). -* Adding a new configuration option? You'll want to modify `bootstrap/src/core/config/flags.rs` - for command line flags and then `bootstrap/src/core/config/config.rs` to copy the flags to the - `Config` struct. -* Adding a sanity check? Take a look at `bootstrap/src/core/sanity.rs`. - -If you make a major change on bootstrap configuration, please add a new entry to -`CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/utils/change_tracker.rs`. - -A 'major change' includes - -* A new option or -* A change in the default options. - -Changes that do not affect contributors to the compiler or users -building rustc from source don't need an update to `CONFIG_CHANGE_HISTORY`. - -If you have any questions, feel free to reach out on the `#t-infra/bootstrap` channel -at [Rust Bootstrap Zulip server][rust-bootstrap-zulip]. When you encounter bugs, -please file issues on the [Rust issue tracker][rust-issue-tracker]. - -[rust-bootstrap-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/t-infra.2Fbootstrap -[rust-issue-tracker]: https://github.com/rust-lang/rust/issues - -## Changelog - -Because we do not release bootstrap with versions, we also do not maintain CHANGELOG files. To -review the changes made to bootstrap, simply run `git log --no-merges --oneline -- src/bootstrap`. diff --git a/standalonex/src/bootstrap.py b/standalonex/src/bootstrap.py deleted file mode 100644 index baad163c..00000000 --- a/standalonex/src/bootstrap.py +++ /dev/null @@ -1,1232 +0,0 @@ -from __future__ import absolute_import, division, print_function -import argparse -import contextlib -import datetime -import hashlib -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile - -from time import time -from multiprocessing import Pool, cpu_count - -try: - import lzma -except ImportError: - lzma = None - -def platform_is_win32(): - return sys.platform == 'win32' - -if platform_is_win32(): - EXE_SUFFIX = ".exe" -else: - EXE_SUFFIX = "" - -def get_cpus(): - if hasattr(os, "sched_getaffinity"): - return len(os.sched_getaffinity(0)) - if hasattr(os, "cpu_count"): - cpus = os.cpu_count() - if cpus is not None: - return cpus - try: - return cpu_count() - except NotImplementedError: - return 1 - - -def eprint(*args, **kwargs): - kwargs["file"] = sys.stderr - print(*args, **kwargs) - - -def get(base, url, path, checksums, verbose=False): - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - try: - if url not in checksums: - raise RuntimeError(("src/stage0 doesn't contain a checksum for {}. " - "Pre-built artifacts might not be available for this " - "target at this time, see https://doc.rust-lang.org/nightly" - "/rustc/platform-support.html for more information.") - .format(url)) - sha256 = checksums[url] - if os.path.exists(path): - if verify(path, sha256, False): - if verbose: - eprint("using already-download file", path) - return - else: - if verbose: - eprint("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) - download(temp_path, "{}/{}".format(base, url), True, verbose) - if not verify(temp_path, sha256, verbose): - raise RuntimeError("failed verification") - if verbose: - eprint("moving {} to {}".format(temp_path, path)) - shutil.move(temp_path, path) - finally: - if os.path.isfile(temp_path): - if verbose: - eprint("removing", temp_path) - os.unlink(temp_path) - -def curl_version(): - m = re.match(bytes("^curl ([0-9]+)\\.([0-9]+)", "utf8"), require(["curl", "-V"])) - if m is None: - return (0, 0) - return (int(m[1]), int(m[2])) - -def download(path, url, probably_big, verbose): - for _ in range(4): - try: - _download(path, url, probably_big, verbose, True) - return - except RuntimeError: - eprint("\nspurious failure, trying again") - _download(path, url, probably_big, verbose, False) - - -def _download(path, url, probably_big, verbose, exception): - # Try to use curl (potentially available on win32 - # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/) - # If an error occurs: - # - If we are on win32 fallback to powershell - # - Otherwise raise the error if appropriate - if probably_big or verbose: - eprint("downloading {}".format(url)) - - try: - if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ: - option = "--progress-bar" - else: - option = "--silent" - # If curl is not present on Win32, we should not sys.exit - # but raise `CalledProcessError` or `OSError` instead - require(["curl", "--version"], exception=platform_is_win32()) - extra_flags = [] - if curl_version() > (7, 70): - extra_flags = [ "--retry-all-errors" ] - # options should be kept in sync with - # src/bootstrap/src/core/download.rs - # for consistency. - # they are also more compreprensivly explained in that file. - run(["curl", option] + extra_flags + [ - # Follow redirect. - "--location", - # timeout if speed is < 10 bytes/sec for > 30 seconds - "--speed-time", "30", "--speed-limit", "10", - # timeout if cannot connect within 30 seconds - "--connect-timeout", "30", - "--output", path, - "--continue-at", "-", - "--retry", "3", "--show-error", "--remote-time", "--fail", url], - verbose=verbose, - exception=True, # Will raise RuntimeError on failure - ) - except (subprocess.CalledProcessError, OSError, RuntimeError): - # see http://serverfault.com/questions/301128/how-to-download - if platform_is_win32(): - run_powershell([ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], - verbose=verbose, - exception=exception) - # Check if the RuntimeError raised by run(curl) should be silenced - elif verbose or exception: - raise - - -def verify(path, expected, verbose): - """Check if the sha256 sum of the given path is valid""" - if verbose: - eprint("verifying", path) - with open(path, "rb") as source: - found = hashlib.sha256(source.read()).hexdigest() - verified = found == expected - if not verified: - eprint("invalid checksum:\n" - " found: {}\n" - " expected: {}".format(found, expected)) - return verified - - -def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): - """Unpack the given tarball file""" - eprint("extracting", tarball) - fname = os.path.basename(tarball).replace(tarball_suffix, "") - with contextlib.closing(tarfile.open(tarball)) as tar: - for member in tar.getnames(): - if "/" not in member: - continue - name = member.replace(fname + "/", "", 1) - if match is not None and not name.startswith(match): - continue - name = name[len(match) + 1:] - - dst_path = os.path.join(dst, name) - if verbose: - eprint(" extracting", member) - tar.extract(member, dst) - src_path = os.path.join(dst, member) - if os.path.isdir(src_path) and os.path.exists(dst_path): - continue - shutil.move(src_path, dst_path) - shutil.rmtree(os.path.join(dst, fname)) - - -def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): - """Run a child program in a new process""" - if verbose: - eprint("running: " + ' '.join(args)) - sys.stdout.flush() - # Ensure that the .exe is used on Windows just in case a Linux ELF has been - # compiled in the same directory. - if os.name == 'nt' and not args[0].endswith('.exe'): - args[0] += '.exe' - # Use Popen here instead of call() as it apparently allows powershell on - # Windows to not lock up waiting for input presumably. - ret = subprocess.Popen(args, **kwargs) - code = ret.wait() - if code != 0: - err = "failed to run: " + ' '.join(args) - if verbose or exception: - raise RuntimeError(err) - # For most failures, we definitely do want to print this error, or the user will have no - # idea what went wrong. But when we've successfully built bootstrap and it failed, it will - # have already printed an error above, so there's no need to print the exact command we're - # running. - if is_bootstrap: - sys.exit(1) - else: - sys.exit(err) - -def run_powershell(script, *args, **kwargs): - """Run a powershell script""" - run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs) - - -def require(cmd, exit=True, exception=False): - '''Run a command, returning its output. - On error, - If `exception` is `True`, raise the error - Otherwise If `exit` is `True`, exit the process - Else return None.''' - try: - return subprocess.check_output(cmd).strip() - except (subprocess.CalledProcessError, OSError) as exc: - if exception: - raise - elif exit: - eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) - eprint("Please make sure it's installed and in the path.") - sys.exit(1) - return None - - - -def format_build_time(duration): - """Return a nicer format for build time - - >>> format_build_time('300') - '0:05:00' - """ - return str(datetime.timedelta(seconds=int(duration))) - - -def default_build_triple(verbose): - """Build triple as in LLVM""" - # If we're on Windows and have an existing `rustc` toolchain, use `rustc --version --verbose` - # to find our host target triple. This fixes an issue with Windows builds being detected - # as GNU instead of MSVC. - # Otherwise, detect it via `uname` - default_encoding = sys.getdefaultencoding() - - if platform_is_win32(): - try: - version = subprocess.check_output(["rustc", "--version", "--verbose"], - stderr=subprocess.DEVNULL) - version = version.decode(default_encoding) - host = next(x for x in version.split('\n') if x.startswith("host: ")) - triple = host.split("host: ")[1] - if verbose: - eprint("detected default triple {} from pre-installed rustc".format(triple)) - return triple - except Exception as e: - if verbose: - eprint("pre-installed rustc not detected: {}".format(e)) - eprint("falling back to auto-detect") - - required = not platform_is_win32() - uname = require(["uname", "-smp"], exit=required) - - # If we do not have `uname`, assume Windows. - if uname is None: - return 'x86_64-pc-windows-msvc' - - kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2) - - # The goal here is to come up with the same triple as LLVM would, - # at least for the subset of platforms we're willing to target. - kerneltype_mapper = { - 'Darwin': 'apple-darwin', - 'DragonFly': 'unknown-dragonfly', - 'FreeBSD': 'unknown-freebsd', - 'Haiku': 'unknown-haiku', - 'NetBSD': 'unknown-netbsd', - 'OpenBSD': 'unknown-openbsd', - 'GNU': 'unknown-hurd', - } - - # Consider the direct transformation first and then the special cases - if kernel in kerneltype_mapper: - kernel = kerneltype_mapper[kernel] - elif kernel == 'Linux': - # Apple doesn't support `-o` so this can't be used in the combined - # uname invocation above - ostype = require(["uname", "-o"], exit=required).decode(default_encoding) - if ostype == 'Android': - kernel = 'linux-android' - else: - kernel = 'unknown-linux-gnu' - elif kernel == 'SunOS': - kernel = 'pc-solaris' - # On Solaris, uname -m will return a machine classification instead - # of a cpu type, so uname -p is recommended instead. However, the - # output from that option is too generic for our purposes (it will - # always emit 'i386' on x86/amd64 systems). As such, isainfo -k - # must be used instead. - cputype = require(['isainfo', '-k']).decode(default_encoding) - # sparc cpus have sun as a target vendor - if 'sparc' in cputype: - kernel = 'sun-solaris' - elif kernel.startswith('MINGW'): - # msys' `uname` does not print gcc configuration, but prints msys - # configuration. so we cannot believe `uname -m`: - # msys1 is always i686 and msys2 is always x86_64. - # instead, msys defines $MSYSTEM which is MINGW32 on i686 and - # MINGW64 on x86_64. - kernel = 'pc-windows-gnu' - cputype = 'i686' - if os.environ.get('MSYSTEM') == 'MINGW64': - cputype = 'x86_64' - elif kernel.startswith('MSYS'): - kernel = 'pc-windows-gnu' - elif kernel.startswith('CYGWIN_NT'): - cputype = 'i686' - if kernel.endswith('WOW64'): - cputype = 'x86_64' - kernel = 'pc-windows-gnu' - elif platform_is_win32(): - # Some Windows platforms might have a `uname` command that returns a - # non-standard string (e.g. gnuwin32 tools returns `windows32`). In - # these cases, fall back to using sys.platform. - return 'x86_64-pc-windows-msvc' - elif kernel == 'AIX': - # `uname -m` returns the machine ID rather than machine hardware on AIX, - # so we are unable to use cputype to form triple. AIX 7.2 and - # above supports 32-bit and 64-bit mode simultaneously and `uname -p` - # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in - # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper - # are not used to infer AIX's triple. - return 'powerpc64-ibm-aix' - else: - err = "unknown OS type: {}".format(kernel) - sys.exit(err) - - if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd': - cputype = subprocess.check_output( - ['uname', '-p']).strip().decode(default_encoding) - cputype_mapper = { - 'BePC': 'i686', - 'aarch64': 'aarch64', - 'aarch64eb': 'aarch64', - 'amd64': 'x86_64', - 'arm64': 'aarch64', - 'i386': 'i686', - 'i486': 'i686', - 'i686': 'i686', - 'i686-AT386': 'i686', - 'i786': 'i686', - 'loongarch64': 'loongarch64', - 'm68k': 'm68k', - 'csky': 'csky', - 'powerpc': 'powerpc', - 'powerpc64': 'powerpc64', - 'powerpc64le': 'powerpc64le', - 'ppc': 'powerpc', - 'ppc64': 'powerpc64', - 'ppc64le': 'powerpc64le', - 'riscv64': 'riscv64gc', - 's390x': 's390x', - 'x64': 'x86_64', - 'x86': 'i686', - 'x86-64': 'x86_64', - 'x86_64': 'x86_64' - } - - # Consider the direct transformation first and then the special cases - if cputype in cputype_mapper: - cputype = cputype_mapper[cputype] - elif cputype in {'xscale', 'arm'}: - cputype = 'arm' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - elif kernel == 'unknown-freebsd': - cputype = processor - kernel = 'unknown-freebsd' - elif cputype == 'armv6l': - cputype = 'arm' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - else: - kernel += 'eabihf' - elif cputype in {'armv7l', 'armv8l'}: - cputype = 'armv7' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - else: - kernel += 'eabihf' - elif cputype == 'mips': - if sys.byteorder == 'big': - cputype = 'mips' - elif sys.byteorder == 'little': - cputype = 'mipsel' - else: - raise ValueError("unknown byteorder: {}".format(sys.byteorder)) - elif cputype == 'mips64': - if sys.byteorder == 'big': - cputype = 'mips64' - elif sys.byteorder == 'little': - cputype = 'mips64el' - else: - raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) - # only the n64 ABI is supported, indicate it - kernel += 'abi64' - elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': - pass - else: - err = "unknown cpu type: {}".format(cputype) - sys.exit(err) - - return "{}-{}".format(cputype, kernel) - - -@contextlib.contextmanager -def output(filepath): - tmp = filepath + '.tmp' - with open(tmp, 'w') as f: - yield f - try: - if os.path.exists(filepath): - os.remove(filepath) # PermissionError/OSError on Win32 if in use - except OSError: - shutil.copy2(tmp, filepath) - os.remove(tmp) - return - os.rename(tmp, filepath) - - -class Stage0Toolchain: - def __init__(self, date, version): - self.date = date - self.version = version - - def channel(self): - return self.version + "-" + self.date - - -class DownloadInfo: - """A helper class that can be pickled into a parallel subprocess""" - - def __init__( - self, - base_download_url, - download_path, - bin_root, - tarball_path, - tarball_suffix, - stage0_data, - pattern, - verbose, - ): - self.base_download_url = base_download_url - self.download_path = download_path - self.bin_root = bin_root - self.tarball_path = tarball_path - self.tarball_suffix = tarball_suffix - self.stage0_data = stage0_data - self.pattern = pattern - self.verbose = verbose - -def download_component(download_info): - if not os.path.exists(download_info.tarball_path): - get( - download_info.base_download_url, - download_info.download_path, - download_info.tarball_path, - download_info.stage0_data, - verbose=download_info.verbose, - ) - -def unpack_component(download_info): - unpack( - download_info.tarball_path, - download_info.tarball_suffix, - download_info.bin_root, - match=download_info.pattern, - verbose=download_info.verbose, - ) - -class FakeArgs: - """Used for unit tests to avoid updating all call sites""" - def __init__(self): - self.build = '' - self.build_dir = '' - self.clean = False - self.verbose = False - self.json_output = False - self.color = 'auto' - self.warnings = 'default' - -class RustBuild(object): - """Provide all the methods required to build Rust""" - def __init__(self, config_toml="", args=None): - if args is None: - args = FakeArgs() - self.git_version = None - self.nix_deps_dir = None - self._should_fix_bins_and_dylibs = None - self.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) - - self.config_toml = config_toml - - self.clean = args.clean - self.json_output = args.json_output - self.verbose = args.verbose - self.color = args.color - self.warnings = args.warnings - - config_verbose_count = self.get_toml('verbose', 'build') - if config_verbose_count is not None: - self.verbose = max(self.verbose, int(config_verbose_count)) - - self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true' - self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true' - - build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build' - self.build_dir = os.path.abspath(build_dir) - - self.stage0_data = parse_stage0_file(os.path.join(self.rust_root, "src", "stage0")) - self.stage0_compiler = Stage0Toolchain( - self.stage0_data["compiler_date"], - self.stage0_data["compiler_version"] - ) - self.download_url = os.getenv("RUSTUP_DIST_SERVER") or self.stage0_data["dist_server"] - - self.build = args.build or self.build_triple() - - - def download_toolchain(self): - """Fetch the build system for Rust, written in Rust - - This method will build a cache directory, then it will fetch the - tarball which has the stage0 compiler used to then bootstrap the Rust - compiler itself. - - Each downloaded tarball is extracted, after that, the script - will move all the content to the right place. - """ - rustc_channel = self.stage0_compiler.version - bin_root = self.bin_root() - - key = self.stage0_compiler.date - is_outdated = self.program_out_of_date(self.rustc_stamp(), key) - need_rustc = self.rustc().startswith(bin_root) and (not os.path.exists(self.rustc()) \ - or is_outdated) - need_cargo = self.cargo().startswith(bin_root) and (not os.path.exists(self.cargo()) \ - or is_outdated) - - if need_rustc or need_cargo: - if os.path.exists(bin_root): - # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's - # running. Kill it. - if platform_is_win32(): - print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain") - regex = '{}\\\\(host|{})\\\\stage0\\\\libexec'.format( - os.path.basename(self.build_dir), - self.build - ) - script = ( - # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if - # the server isn't running. - 'Get-Process | ' + - 'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' + - 'Where-Object {{$_.Path -match "{}"}} |'.format(regex) + - 'Stop-Process' - ) - run_powershell([script]) - shutil.rmtree(bin_root) - - cache_dst = (self.get_toml('bootstrap-cache-path', 'build') or - os.path.join(self.build_dir, "cache")) - - rustc_cache = os.path.join(cache_dst, key) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) - - tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' - - toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - - tarballs_to_download = [] - - if need_rustc: - tarballs_to_download.append( - ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)) - ) - tarballs_to_download.append(("rustc-{}".format(toolchain_suffix), "rustc")) - - if need_cargo: - tarballs_to_download.append(("cargo-{}".format(toolchain_suffix), "cargo")) - - tarballs_download_info = [ - DownloadInfo( - base_download_url=self.download_url, - download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), - bin_root=self.bin_root(), - tarball_path=os.path.join(rustc_cache, filename), - tarball_suffix=tarball_suffix, - stage0_data=self.stage0_data, - pattern=pattern, - verbose=self.verbose, - ) - for filename, pattern in tarballs_to_download - ] - - # Download the components serially to show the progress bars properly. - for download_info in tarballs_download_info: - download_component(download_info) - - # Unpack the tarballs in parallle. - # In Python 2.7, Pool cannot be used as a context manager. - pool_size = min(len(tarballs_download_info), get_cpus()) - if self.verbose: - print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') - p = Pool(pool_size) - try: - # FIXME: A cheap workaround for https://github.com/rust-lang/rust/issues/125578, - # remove this once the issue is closed. - bootstrap_build_artifacts = os.path.join(self.bootstrap_out(), "debug") - if os.path.exists(bootstrap_build_artifacts): - shutil.rmtree(bootstrap_build_artifacts) - - p.map(unpack_component, tarballs_download_info) - finally: - p.close() - p.join() - - if self.should_fix_bins_and_dylibs(): - self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) - - self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) - self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) - self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root)) - lib_dir = "{}/lib".format(bin_root) - rustlib_bin_dir = "{}/rustlib/{}/bin".format(lib_dir, self.build) - self.fix_bin_or_dylib("{}/rust-lld".format(rustlib_bin_dir)) - self.fix_bin_or_dylib("{}/gcc-ld/ld.lld".format(rustlib_bin_dir)) - for lib in os.listdir(lib_dir): - # .so is not necessarily the suffix, there can be version numbers afterwards. - if ".so" in lib: - elf_path = os.path.join(lib_dir, lib) - with open(elf_path, "rb") as f: - magic = f.read(4) - # Patchelf will skip non-ELF files, but issue a warning. - if magic == b"\x7fELF": - self.fix_bin_or_dylib(elf_path) - - with output(self.rustc_stamp()) as rust_stamp: - rust_stamp.write(key) - - def should_fix_bins_and_dylibs(self): - """Whether or not `fix_bin_or_dylib` needs to be run; can only be True - on NixOS or if config.toml has `build.patch-binaries-for-nix` set. - """ - if self._should_fix_bins_and_dylibs is not None: - return self._should_fix_bins_and_dylibs - - def get_answer(): - default_encoding = sys.getdefaultencoding() - try: - ostype = subprocess.check_output( - ['uname', '-s']).strip().decode(default_encoding) - except subprocess.CalledProcessError: - return False - except OSError as reason: - if getattr(reason, 'winerror', None) is not None: - return False - raise reason - - if ostype != "Linux": - return False - - # If the user has explicitly indicated whether binaries should be - # patched for Nix, then don't check for NixOS. - if self.get_toml("patch-binaries-for-nix", "build") == "true": - return True - if self.get_toml("patch-binaries-for-nix", "build") == "false": - return False - - # Use `/etc/os-release` instead of `/etc/NIXOS`. - # The latter one does not exist on NixOS when using tmpfs as root. - try: - with open("/etc/os-release", "r") as f: - is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') - for ln in f) - except FileNotFoundError: - is_nixos = False - - # If not on NixOS, then warn if user seems to be atop Nix shell - if not is_nixos: - in_nix_shell = os.getenv('IN_NIX_SHELL') - if in_nix_shell: - eprint("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell), - "you may need to set `patch-binaries-for-nix=true` in config.toml") - - return is_nixos - - answer = self._should_fix_bins_and_dylibs = get_answer() - if answer: - eprint("INFO: You seem to be using Nix.") - return answer - - def fix_bin_or_dylib(self, fname): - """Modifies the interpreter section of 'fname' to fix the dynamic linker, - or the RPATH section, to fix the dynamic library search path - - This method is only required on NixOS and uses the PatchELF utility to - change the interpreter/RPATH of ELF executables. - - Please see https://nixos.org/patchelf.html for more information - """ - assert self._should_fix_bins_and_dylibs is True - eprint("attempting to patch", fname) - - # Only build `.nix-deps` once. - nix_deps_dir = self.nix_deps_dir - if not nix_deps_dir: - # Run `nix-build` to "build" each dependency (which will likely reuse - # the existing `/nix/store` copy, or at most download a pre-built copy). - # - # Importantly, we create a gc-root called `.nix-deps` in the `build/` - # directory, but still reference the actual `/nix/store` path in the rpath - # as it makes it significantly more robust against changes to the location of - # the `.nix-deps` location. - # - # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - # zlib: Needed as a system dependency of `libLLVM-*.so`. - # patchelf: Needed for patching ELF binaries (see doc comment above). - nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") - nix_expr = ''' - with (import {}); - symlinkJoin { - name = "rust-stage0-dependencies"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - ''' - try: - subprocess.check_output([ - "nix-build", "-E", nix_expr, "-o", nix_deps_dir, - ]) - except subprocess.CalledProcessError as reason: - eprint("WARNING: failed to call nix-build:", reason) - return - self.nix_deps_dir = nix_deps_dir - - patchelf = "{}/bin/patchelf".format(nix_deps_dir) - rpath_entries = [ - os.path.join(os.path.realpath(nix_deps_dir), "lib") - ] - patchelf_args = ["--add-rpath", ":".join(rpath_entries)] - if ".so" not in fname: - # Finally, set the correct .interp for binaries - with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: - patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] - - try: - subprocess.check_output([patchelf] + patchelf_args + [fname]) - except subprocess.CalledProcessError as reason: - eprint("WARNING: failed to call patchelf:", reason) - return - - def rustc_stamp(self): - """Return the path for .rustc-stamp at the given stage - - >>> rb = RustBuild() - >>> rb.build = "host" - >>> rb.build_dir = "build" - >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp") - >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp() - """ - return os.path.join(self.bin_root(), '.rustc-stamp') - - def program_out_of_date(self, stamp_path, key): - """Check if the given program stamp is out of date""" - if not os.path.exists(stamp_path) or self.clean: - return True - with open(stamp_path, 'r') as stamp: - return key != stamp.read() - - def bin_root(self): - """Return the binary root directory for the given stage - - >>> rb = RustBuild() - >>> rb.build = "devel" - >>> expected = os.path.abspath(os.path.join("build", "devel", "stage0")) - >>> assert rb.bin_root() == expected, rb.bin_root() - """ - subdir = "stage0" - return os.path.join(self.build_dir, self.build, subdir) - - def get_toml(self, key, section=None): - """Returns the value of the given key in config.toml, otherwise returns None - - >>> rb = RustBuild() - >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' - >>> rb.get_toml("key2") - 'value2' - - If the key does not exist, the result is None: - - >>> rb.get_toml("key3") is None - True - - Optionally also matches the section the key appears in - - >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' - >>> rb.get_toml('key', 'a') - 'value1' - >>> rb.get_toml('key', 'b') - 'value2' - >>> rb.get_toml('key', 'c') is None - True - - >>> rb.config_toml = 'key1 = true' - >>> rb.get_toml("key1") - 'true' - """ - return RustBuild.get_toml_static(self.config_toml, key, section) - - @staticmethod - def get_toml_static(config_toml, key, section=None): - cur_section = None - for line in config_toml.splitlines(): - section_match = re.match(r'^\s*\[(.*)\]\s*$', line) - if section_match is not None: - cur_section = section_match.group(1) - - match = re.match(r'^{}\s*=(.*)$'.format(key), line) - if match is not None: - value = match.group(1) - if section is None or section == cur_section: - return RustBuild.get_string(value) or value.strip() - return None - - def cargo(self): - """Return config path for cargo""" - return self.program_config('cargo') - - def rustc(self): - """Return config path for rustc""" - return self.program_config('rustc') - - def program_config(self, program): - """Return config path for the given program at the given stage - - >>> rb = RustBuild() - >>> rb.config_toml = 'rustc = "rustc"\\n' - >>> rb.program_config('rustc') - 'rustc' - >>> rb.config_toml = '' - >>> cargo_path = rb.program_config('cargo') - >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), - ... "bin", "cargo") - True - """ - config = self.get_toml(program) - if config: - return os.path.expanduser(config) - return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX)) - - @staticmethod - def get_string(line): - """Return the value between double quotes - - >>> RustBuild.get_string(' "devel" ') - 'devel' - >>> RustBuild.get_string(" 'devel' ") - 'devel' - >>> RustBuild.get_string('devel') is None - True - >>> RustBuild.get_string(' "devel ') - '' - """ - start = line.find('"') - if start != -1: - end = start + 1 + line[start + 1:].find('"') - return line[start + 1:end] - start = line.find('\'') - if start != -1: - end = start + 1 + line[start + 1:].find('\'') - return line[start + 1:end] - return None - - def bootstrap_out(self): - """Return the path of the bootstrap build artifacts - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap") - True - """ - return os.path.join(self.build_dir, "bootstrap") - - def bootstrap_binary(self): - """Return the path of the bootstrap binary - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", - ... "debug", "bootstrap") - True - """ - return os.path.join(self.bootstrap_out(), "debug", "bootstrap") - - def build_bootstrap(self): - """Build bootstrap""" - env = os.environ.copy() - if "GITHUB_ACTIONS" in env: - print("::group::Building bootstrap") - else: - eprint("Building bootstrap") - - args = self.build_bootstrap_cmd(env) - # Run this from the source directory so cargo finds .cargo/config - run(args, env=env, verbose=self.verbose, cwd=self.rust_root) - - if "GITHUB_ACTIONS" in env: - print("::endgroup::") - - def build_bootstrap_cmd(self, env): - """For tests.""" - build_dir = os.path.join(self.build_dir, "bootstrap") - if self.clean and os.path.exists(build_dir): - shutil.rmtree(build_dir) - # `CARGO_BUILD_TARGET` breaks bootstrap build. - # See also: . - if "CARGO_BUILD_TARGET" in env: - del env["CARGO_BUILD_TARGET"] - env["CARGO_TARGET_DIR"] = build_dir - env["RUSTC"] = self.rustc() - env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LD_LIBRARY_PATH"]) \ - if "LD_LIBRARY_PATH" in env else "" - env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ - if "DYLD_LIBRARY_PATH" in env else "" - env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LIBRARY_PATH"]) \ - if "LIBRARY_PATH" in env else "" - env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LIBPATH"]) \ - if "LIBPATH" in env else "" - - # Export Stage0 snapshot compiler related env variables - build_section = "target.{}".format(self.build) - host_triple_sanitized = self.build.replace("-", "_") - var_data = { - "CC": "cc", "CXX": "cxx", "LD": "linker", "AR": "ar", "RANLIB": "ranlib" - } - for var_name, toml_key in var_data.items(): - toml_val = self.get_toml(toml_key, build_section) - if toml_val is not None: - env["{}_{}".format(var_name, host_triple_sanitized)] = toml_val - - # In src/etc/rust_analyzer_settings.json, we configure rust-analyzer to - # pass RUSTC_BOOTSTRAP=1 to all cargo invocations because the standard - # library uses unstable Cargo features. Without RUSTC_BOOTSTRAP, - # rust-analyzer would fail to fetch workspace layout when the system's - # default toolchain is not nightly. - # - # But that setting has the collateral effect of rust-analyzer also - # passing RUSTC_BOOTSTRAP=1 to all x.py invocations too (the various - # overrideCommand). - # - # Set a consistent RUSTC_BOOTSTRAP=1 here to prevent spurious rebuilds - # of bootstrap when rust-analyzer x.py invocations are interleaved with - # handwritten ones on the command line. - env["RUSTC_BOOTSTRAP"] = "1" - - # If any of RUSTFLAGS or RUSTFLAGS_BOOTSTRAP are present and nonempty, - # we allow arbitrary compiler flags in there, including unstable ones - # such as `-Zthreads=8`. - # - # But if there aren't custom flags being passed to bootstrap, then we - # cancel the RUSTC_BOOTSTRAP=1 from above by passing `-Zallow-features=` - # to ensure unstable language or library features do not accidentally - # get introduced into bootstrap over time. Distros rely on being able to - # compile bootstrap with a variety of their toolchains, not necessarily - # the same as Rust's CI uses. - if env.get("RUSTFLAGS", "") or env.get("RUSTFLAGS_BOOTSTRAP", ""): - # Preserve existing RUSTFLAGS. - env.setdefault("RUSTFLAGS", "") - else: - env["RUSTFLAGS"] = "-Zallow-features=" - - target_features = [] - if self.get_toml("crt-static", build_section) == "true": - target_features += ["+crt-static"] - elif self.get_toml("crt-static", build_section) == "false": - target_features += ["-crt-static"] - if target_features: - env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) - target_linker = self.get_toml("linker", build_section) - if target_linker is not None: - env["RUSTFLAGS"] += " -C linker=" + target_linker - # When changing this list, also update the corresponding list in `Builder::cargo` - # in `src/bootstrap/src/core/builder.rs`. - env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" - if self.warnings == "default": - deny_warnings = self.get_toml("deny-warnings", "rust") != "false" - else: - deny_warnings = self.warnings == "deny" - if deny_warnings: - env["RUSTFLAGS"] += " -Dwarnings" - - # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. - # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of - # RUSTFLAGS to be actually effective (e.g., if we have `-Dwarnings` in - # RUSTFLAGS, passing `-Awarnings` from RUSTFLAGS_BOOTSTRAP should override it). - if "RUSTFLAGS_BOOTSTRAP" in env: - env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] - - env["PATH"] = os.path.join(self.bin_root(), "bin") + \ - os.pathsep + env["PATH"] - if not os.path.isfile(self.cargo()): - raise Exception("no cargo executable found at `{}`".format( - self.cargo())) - args = [self.cargo(), "build", "--manifest-path", - os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] - args.extend("--verbose" for _ in range(self.verbose)) - if self.use_locked_deps: - args.append("--locked") - if self.use_vendored_sources: - args.append("--frozen") - if self.get_toml("metrics", "build"): - args.append("--features") - args.append("build-metrics") - if self.json_output: - args.append("--message-format=json") - if self.color == "always": - args.append("--color=always") - elif self.color == "never": - args.append("--color=never") - try: - args += env["CARGOFLAGS"].split() - except KeyError: - pass - - return args - - def build_triple(self): - """Build triple as in LLVM - - Note that `default_build_triple` is moderately expensive, - so use `self.build` where possible. - """ - config = self.get_toml('build') - return config or default_build_triple(self.verbose) - - def check_vendored_status(self): - """Check that vendoring is configured properly""" - # keep this consistent with the equivalent check in bootstrap: - # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 - if 'SUDO_USER' in os.environ and not self.use_vendored_sources: - if os.getuid() == 0: - self.use_vendored_sources = True - eprint('INFO: looks like you\'re trying to run this command as root') - eprint(' and so in order to preserve your $HOME this will now') - eprint(' use vendored sources by default.') - - cargo_dir = os.path.join(self.rust_root, '.cargo') - if self.use_vendored_sources: - vendor_dir = os.path.join(self.rust_root, 'vendor') - if not os.path.exists(vendor_dir): - eprint('ERROR: vendoring required, but vendor directory does not exist.') - eprint(' Run `x.py vendor` to initialize the vendor directory.') - eprint(' Alternatively, use the pre-vendored `rustc-src` dist component.') - eprint(' To get a stable/beta/nightly version, download it from: ') - eprint(' ' - 'https://forge.rust-lang.org/infra/other-installation-methods.html#source-code') - eprint(' To get a specific commit version, download it using the below URL,') - eprint(' replacing with a specific commit checksum: ') - eprint(' ' - 'https://ci-artifacts.rust-lang.org/rustc-builds//rustc-nightly-src.tar.xz') - eprint(' Once you have the source downloaded, place the vendor directory') - eprint(' from the archive in the root of the rust project.') - raise Exception("{} not found".format(vendor_dir)) - - if not os.path.exists(cargo_dir): - eprint('ERROR: vendoring required, but .cargo/config does not exist.') - raise Exception("{} not found".format(cargo_dir)) - -def parse_args(args): - """Parse the command line arguments that the python script needs.""" - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('-h', '--help', action='store_true') - parser.add_argument('--config') - parser.add_argument('--build-dir') - parser.add_argument('--build') - parser.add_argument('--color', choices=['always', 'never', 'auto']) - parser.add_argument('--clean', action='store_true') - parser.add_argument('--json-output', action='store_true') - parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default') - parser.add_argument('-v', '--verbose', action='count', default=0) - - return parser.parse_known_args(args)[0] - -def parse_stage0_file(path): - result = {} - with open(path, 'r') as file: - for line in file: - line = line.strip() - if line and not line.startswith('#'): - key, value = line.split('=', 1) - result[key.strip()] = value.strip() - return result - -def bootstrap(args): - """Configure, fetch, build and run the initial bootstrap""" - rust_root = os.path.abspath(os.path.join(__file__, '../../..')) - - if not os.path.exists(os.path.join(rust_root, '.git')) and \ - os.path.exists(os.path.join(rust_root, '.github')): - eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " - "git clone nor distributed tarball.\nThis build may fail due to missing submodules " - "unless you put them in place manually.") - - # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, - # then `config.toml` in the root directory. - toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') - using_default_path = toml_path is None - if using_default_path: - toml_path = 'config.toml' - if not os.path.exists(toml_path): - toml_path = os.path.join(rust_root, toml_path) - - # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - # but not if `config.toml` hasn't been created. - if not using_default_path or os.path.exists(toml_path): - with open(toml_path) as config: - config_toml = config.read() - else: - config_toml = '' - - profile = RustBuild.get_toml_static(config_toml, "profile") - is_non_git_source = not os.path.exists(os.path.join(rust_root, ".git")) - - if profile is None and is_non_git_source: - profile = "dist" - - if profile is not None: - # Allows creating alias for profile names, allowing - # profiles to be renamed while maintaining back compatibility - # Keep in sync with `profile_aliases` in config.rs - profile_aliases = { - "user": "dist" - } - include_file = 'config.{}.toml'.format(profile_aliases.get(profile) or profile) - include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults') - include_path = os.path.join(include_dir, include_file) - - if not os.path.exists(include_path): - raise Exception("Unrecognized config profile '{}'. Check src/bootstrap/defaults" - " for available options.".format(profile)) - - # HACK: This works because `self.get_toml()` returns the first match it finds for a - # specific key, so appending our defaults at the end allows the user to override them - with open(include_path) as included_toml: - config_toml += os.linesep + included_toml.read() - - # Configure initial bootstrap - build = RustBuild(config_toml, args) - build.check_vendored_status() - - if not os.path.exists(build.build_dir): - os.makedirs(build.build_dir) - - # Fetch/build the bootstrap - build.download_toolchain() - sys.stdout.flush() - build.build_bootstrap() - sys.stdout.flush() - - # Run the bootstrap - args = [build.bootstrap_binary()] - args.extend(sys.argv[1:]) - env = os.environ.copy() - env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) - env["BOOTSTRAP_PYTHON"] = sys.executable - run(args, env=env, verbose=build.verbose, is_bootstrap=True) - - -def main(): - """Entry point for the bootstrap process""" - start_time = time() - - # x.py help ... - if len(sys.argv) > 1 and sys.argv[1] == 'help': - sys.argv[1] = '-h' - - args = parse_args(sys.argv) - help_triggered = args.help or len(sys.argv) == 1 - - # If the user is asking for help, let them know that the whole download-and-build - # process has to happen before anything is printed out. - if help_triggered: - eprint( - "INFO: Downloading and building bootstrap before processing --help command.\n" - " See src/bootstrap/README.md for help with common commands.") - - exit_code = 0 - success_word = "successfully" - try: - bootstrap(args) - except (SystemExit, KeyboardInterrupt) as error: - if hasattr(error, 'code') and isinstance(error.code, int): - exit_code = error.code - else: - exit_code = 1 - eprint(error) - success_word = "unsuccessfully" - - if not help_triggered: - eprint("Build completed", success_word, "in", format_build_time(time() - start_time)) - sys.exit(exit_code) - - -if __name__ == '__main__': - main() diff --git a/standalonex/src/bootstrap/Cargo.lock b/standalonex/src/bootstrap/Cargo.lock index efcac4f0..c5e3f46e 100644 --- a/standalonex/src/bootstrap/Cargo.lock +++ b/standalonex/src/bootstrap/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -13,15 +13,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "block-buffer" @@ -36,12 +36,16 @@ dependencies = [ name = "bootstrap" version = "0.0.0" dependencies = [ + "bootstrap-config-utils", "build_helper", "cc", "clap", "clap_complete", "cmake", + "config_core", + "config_macros", "fd-lock", + "globset", "home", "ignore", "junction", @@ -63,11 +67,15 @@ dependencies = [ "xz2", ] +[[package]] +name = "bootstrap-config-utils" +version = "0.1.0" + [[package]] name = "bstr" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata", @@ -93,15 +101,15 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "clap" -version = "4.5.18" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623" dependencies = [ "clap_builder", "clap_derive", @@ -109,9 +117,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0" dependencies = [ "anstyle", "clap_lex", @@ -119,18 +127,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.29" +version = "4.5.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e" +checksum = "2348487adcd4631696ced64ccdb40d38ac4d31cae7f2eec8817fcea1b9d1c43c" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -140,9 +148,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" @@ -153,6 +161,24 @@ dependencies = [ "cc", ] +[[package]] +name = "config_core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "config_macros" +version = "0.1.0" +dependencies = [ + "config_core", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -161,18 +187,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -189,9 +215,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -221,42 +247,42 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "fd-lock" -version = "4.0.2" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "filetime" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -264,9 +290,9 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" dependencies = [ "aho-corasick", "bstr", @@ -283,11 +309,11 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -308,31 +334,31 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "junction" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bbdfd737a243da3dfc1f99ee8d6e166480f17ab4ac84d7c34aacd73fc7bd16" +checksum = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8" dependencies = [ "scopeguard", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] name = "libc" -version = "0.2.159" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags", "libc", @@ -341,15 +367,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "log" -version = "0.4.22" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lzma-sys" @@ -364,9 +390,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "ntapi" @@ -379,9 +405,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -398,9 +424,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pretty_assertions" @@ -414,36 +440,36 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -452,28 +478,28 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" -version = "0.38.37" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -492,24 +518,34 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -518,21 +554,22 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -547,9 +584,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" dependencies = [ "proc-macro2", "quote", @@ -571,9 +608,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" dependencies = [ "filetime", "libc", @@ -600,15 +637,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "version_check" @@ -644,11 +681,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -664,7 +701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core 0.52.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -674,7 +711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ "windows-core 0.57.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -683,7 +720,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -695,7 +732,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-result", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -720,31 +757,46 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-result" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets", + "windows-link", ] [[package]] @@ -753,14 +805,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -769,56 +838,103 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "xattr" -version = "1.3.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "linux-raw-sys", "rustix", ] diff --git a/standalonex/src/bootstrap/Cargo.toml b/standalonex/src/bootstrap/Cargo.toml index 1bb1a06e..afc500c5 100644 --- a/standalonex/src/bootstrap/Cargo.toml +++ b/standalonex/src/bootstrap/Cargo.toml @@ -33,19 +33,27 @@ name = "sccache-plus-cl" path = "src/bin/sccache-plus-cl.rs" test = false +[[bin]] +name = "nix_bootstrap" +path = "src/bin/nix_bootstrap.rs" +test = false + [dependencies] +ignore = "=0.4.23" +globset = "=0.4.16" # Most of the time updating these dependencies requires modifications to the # bootstrap codebase(e.g., https://github.com/rust-lang/rust/issues/124565); # otherwise, some targets will fail. That's why these dependencies are explicitly pinned. cc = "=1.1.22" cmake = "=0.1.48" -build_helper = { path = "../build_helper" } + + clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } clap_complete = "4.4" fd-lock = "4.0" home = "0.5" -ignore = "0.4" + libc = "0.2" object = { version = "0.36.3", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } opener = "0.5" @@ -62,6 +70,11 @@ toml = "0.5" walkdir = "2.4" xz2 = "0.1" +config_core = { path = "src/core/config_crates/config_core" } +config_macros = { path = "src/core/config_crates/config_macros" } +bootstrap-config-utils = { path = "src/core/config_utils" } +bootstrap-test-utils = { path = "src/core/build_steps/test_utils" } + # Dependencies needed by the build-metrics feature sysinfo = { version = "0.31.2", default-features = false, optional = true, features = ["system"] } @@ -90,6 +103,4 @@ debug = 0 [profile.dev.package] # Only use debuginfo=1 to further reduce compile times. -bootstrap.debug = 1 - -[workspace] \ No newline at end of file +bootstrap.debug = 1 \ No newline at end of file diff --git a/standalonex/src/bootstrap/README.md b/standalonex/src/bootstrap/README.md deleted file mode 100644 index f036603e..00000000 --- a/standalonex/src/bootstrap/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# Bootstrapping Rust - -This README is aimed at helping to explain how Rust is bootstrapped, -and some of the technical details of the bootstrap build system. - -Note that this README only covers internal information, not how to use the tool. -Please check [bootstrapping dev guide][bootstrapping-dev-guide] for further information. - -[bootstrapping-dev-guide]: https://rustc-dev-guide.rust-lang.org/building/bootstrapping/intro.html - -## Introduction - -The build system defers most of the complicated logic of managing invocations -of rustc and rustdoc to Cargo itself. However, moving through various stages -and copying artifacts is still necessary for it to do. Each time bootstrap -is invoked, it will iterate through the list of predefined steps and execute -each serially in turn if it matches the paths passed or is a default rule. -For each step, bootstrap relies on the step internally being incremental and -parallel. Note, though, that the `-j` parameter to bootstrap gets forwarded -to appropriate test harnesses and such. - -## Build phases - -Bootstrap build system goes through a few phases to actually build the -compiler. What actually happens when you invoke bootstrap is: - -1. The entry point script (`x` for unix like systems, `x.ps1` for windows systems, - `x.py` cross-platform) is run. This script is responsible for downloading the stage0 - compiler/Cargo binaries, and it then compiles the build system itself (this folder). - Finally, it then invokes the actual `bootstrap` binary build system. -2. In Rust, `bootstrap` will slurp up all configuration, perform a number of - sanity checks (whether compilers exist, for example), and then start building the - stage0 artifacts. -3. The stage0 `cargo`, downloaded earlier, is used to build the standard library - and the compiler, and then these binaries are then copied to the `stage1` - directory. That compiler is then used to generate the stage1 artifacts which - are then copied to the stage2 directory, and then finally, the stage2 - artifacts are generated using that compiler. - -The goal of each stage is to (a) leverage Cargo as much as possible and failing -that (b) leverage Rust as much as possible! - -## Directory Layout - -This build system houses all output under the `build` directory, which looks -like this: - -```sh -# Root folder of all output. Everything is scoped underneath here -build/ - - # Location where the stage0 compiler downloads are all cached. This directory - # only contains the tarballs themselves, as they're extracted elsewhere. - cache/ - 2015-12-19/ - 2016-01-15/ - 2016-01-21/ - ... - - # Output directory for building this build system itself. The stage0 - # cargo/rustc are used to build the build system into this location. - bootstrap/ - debug/ - release/ - - # Output of the dist-related steps like dist-std, dist-rustc, and dist-docs - dist/ - - # Temporary directory used for various input/output as part of various stages - tmp/ - - # Each remaining directory is scoped by the "host" triple of compilation at - # hand. - x86_64-unknown-linux-gnu/ - - # The build artifacts for the `compiler-rt` library for the target that - # this folder is under. The exact layout here will likely depend on the - # platform, and this is also built with CMake, so the build system is - # also likely different. - compiler-rt/ - build/ - - # Output folder for LLVM if it is compiled for this target - llvm/ - - # build folder (e.g. the platform-specific build system). Like with - # compiler-rt, this is compiled with CMake - build/ - - # Installation of LLVM. Note that we run the equivalent of 'make install' - # for LLVM, to setup these folders. - bin/ - lib/ - include/ - share/ - ... - - # Output folder for all documentation of this target. This is what's filled - # in whenever the `doc` step is run. - doc/ - - # Output for all compiletest-based test suites - test/ - ui/ - debuginfo/ - ... - - # Location where the stage0 Cargo and Rust compiler are unpacked. This - # directory is purely an extracted and overlaid tarball of these two (done - # by the bootstrap Python script). In theory, the build system does not - # modify anything under this directory afterwards. - stage0/ - - # These to-build directories are the cargo output directories for builds of - # the standard library, the test system, the compiler, and various tools, - # respectively. Internally, these may also - # have other target directories, which represent artifacts being compiled - # from the host to the specified target. - # - # Essentially, each of these directories is filled in by one `cargo` - # invocation. The build system instruments calling Cargo in the right order - # with the right variables to ensure that these are filled in correctly. - stageN-std/ - stageN-test/ - stageN-rustc/ - stageN-tools/ - - # This is a special case of the above directories, **not** filled in via - # Cargo but rather the build system itself. The stage0 compiler already has - # a set of target libraries for its own host triple (in its own sysroot) - # inside of stage0/. When we run the stage0 compiler to bootstrap more - # things, however, we don't want to use any of these libraries (as those are - # the ones that we're building). So essentially, when the stage1 compiler is - # being compiled (e.g. after libstd has been built), *this* is used as the - # sysroot for the stage0 compiler being run. - # - # Basically, this directory is just a temporary artifact used to configure the - # stage0 compiler to ensure that the libstd that we just built is used to - # compile the stage1 compiler. - stage0-sysroot/lib/ - - # These output directories are intended to be standalone working - # implementations of the compiler (corresponding to each stage). The build - # system will link (using hard links) output from stageN-{std,rustc} into - # each of these directories. - # - # In theory these are working rustc sysroot directories, meaning there is - # no extra build output in these directories. - stage1/ - stage2/ - stage3/ -``` - -## Extending bootstrap - -When you use bootstrap, you'll call it through the entry point script -(`x`, `x.ps1`, or `x.py`). However, most of the code lives in `src/bootstrap`. -`bootstrap` has a difficult problem: it is written in Rust, but yet it is run -before the Rust compiler is built! To work around this, there are two components -of bootstrap: the main one written in rust, and `bootstrap.py`. `bootstrap.py` -is what gets run by entry point script. It takes care of downloading the `stage0` -compiler, which will then build the bootstrap binary written in Rust. - -Because there are two separate codebases behind `x.py`, they need to -be kept in sync. In particular, both `bootstrap.py` and the bootstrap binary -parse `config.toml` and read the same command line arguments. `bootstrap.py` -keeps these in sync by setting various environment variables, and the -programs sometimes have to add arguments that are explicitly ignored, to be -read by the other. - -Some general areas that you may be interested in modifying are: - -* Adding a new build tool? Take a look at `bootstrap/src/core/build_steps/tool.rs` - for examples of other tools. -* Adding a new compiler crate? Look no further! Adding crates can be done by - adding a new directory with `Cargo.toml`, followed by configuring all - `Cargo.toml` files accordingly. -* Adding a new dependency from crates.io? This should just work inside the - compiler artifacts stage (everything other than libtest and libstd). -* Adding a new configuration option? You'll want to modify `bootstrap/src/core/config/flags.rs` - for command line flags and then `bootstrap/src/core/config/config.rs` to copy the flags to the - `Config` struct. -* Adding a sanity check? Take a look at `bootstrap/src/core/sanity.rs`. - -If you make a major change on bootstrap configuration, please add a new entry to -`CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/utils/change_tracker.rs`. - -A 'major change' includes - -* A new option or -* A change in the default options. - -Changes that do not affect contributors to the compiler or users -building rustc from source don't need an update to `CONFIG_CHANGE_HISTORY`. - -If you have any questions, feel free to reach out on the `#t-infra/bootstrap` channel -at [Rust Bootstrap Zulip server][rust-bootstrap-zulip]. When you encounter bugs, -please file issues on the [Rust issue tracker][rust-issue-tracker]. - -[rust-bootstrap-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/t-infra.2Fbootstrap -[rust-issue-tracker]: https://github.com/rust-lang/rust/issues - -## Changelog - -Because we do not release bootstrap with versions, we also do not maintain CHANGELOG files. To -review the changes made to bootstrap, simply run `git log --no-merges --oneline -- src/bootstrap`. diff --git a/standalonex/src/bootstrap/__pycache__/bootstrap.cpython-313.pyc b/standalonex/src/bootstrap/__pycache__/bootstrap.cpython-313.pyc deleted file mode 100644 index 5747b22a..00000000 Binary files a/standalonex/src/bootstrap/__pycache__/bootstrap.cpython-313.pyc and /dev/null differ diff --git a/standalonex/src/bootstrap/bootstrap.py b/standalonex/src/bootstrap/bootstrap.py deleted file mode 100644 index be503a9a..00000000 --- a/standalonex/src/bootstrap/bootstrap.py +++ /dev/null @@ -1,1251 +0,0 @@ -import json -import os -from __future__ import absolute_import, division, print_function -import argparse -import contextlib -import datetime -import hashlib -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile - -from time import time -from multiprocessing import Pool, cpu_count - -try: - import lzma -except ImportError: - lzma = None - -def platform_is_win32(): - return sys.platform == 'win32' - -if platform_is_win32(): - EXE_SUFFIX = ".exe" -else: - EXE_SUFFIX = "" - -def get_cpus(): - if hasattr(os, "sched_getaffinity"): - return len(os.sched_getaffinity(0)) - if hasattr(os, "cpu_count"): - cpus = os.cpu_count() - if cpus is not None: - return cpus - try: - return cpu_count() - except NotImplementedError: - return 1 - - -def eprint(*args, **kwargs): - kwargs["file"] = sys.stderr - print(*args, **kwargs) - - -def get(base, url, path, checksums, verbose=False): - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - try: - if url not in checksums: - raise RuntimeError(("src/stage0 doesn't contain a checksum for {}. " - "Pre-built artifacts might not be available for this " - "target at this time, see https://doc.rust-lang.org/nightly" - "/rustc/platform-support.html for more information.") - .format(url)) - sha256 = checksums[url] - if os.path.exists(path): - if verify(path, sha256, False): - if verbose: - eprint("using already-download file", path) - return - else: - if verbose: - eprint("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) - download(temp_path, "{}/{}".format(base, url), True, verbose) - if not verify(temp_path, sha256, verbose): - raise RuntimeError("failed verification") - if verbose: - eprint("moving {} to {}".format(temp_path, path)) - shutil.move(temp_path, path) - finally: - if os.path.isfile(temp_path): - if verbose: - eprint("removing", temp_path) - os.unlink(temp_path) - -def curl_version(): - m = re.match(bytes("^curl ([0-9]+)\\.([0-9]+)", "utf8"), require(["curl", "-V"])) - if m is None: - return (0, 0) - return (int(m[1]), int(m[2])) - -def download(path, url, probably_big, verbose): - for _ in range(4): - try: - _download(path, url, probably_big, verbose, True) - return - except RuntimeError: - eprint("\nspurious failure, trying again") - _download(path, url, probably_big, verbose, False) - - -def _download(path, url, probably_big, verbose, exception): - # Try to use curl (potentially available on win32 - # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/) - # If an error occurs: - # - If we are on win32 fallback to powershell - # - Otherwise raise the error if appropriate - if probably_big or verbose: - eprint("downloading {}".format(url)) - - try: - if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ: - option = "--progress-bar" - else: - option = "--silent" - # If curl is not present on Win32, we should not sys.exit - # but raise `CalledProcessError` or `OSError` instead - require(["curl", "--version"], exception=platform_is_win32()) - extra_flags = [] - if curl_version() > (7, 70): - extra_flags = [ "--retry-all-errors" ] - # options should be kept in sync with - # src/bootstrap/src/core/download.rs - # for consistency. - # they are also more compreprensivly explained in that file. - run(["curl", option] + extra_flags + [ - # Follow redirect. - "--location", - # timeout if speed is < 10 bytes/sec for > 30 seconds - "--speed-time", "30", "--speed-limit", "10", - # timeout if cannot connect within 30 seconds - "--connect-timeout", "30", - "--output", path, - "--continue-at", "-", - "--retry", "3", "--show-error", "--remote-time", "--fail", url], - verbose=verbose, - exception=True, # Will raise RuntimeError on failure - ) - except (subprocess.CalledProcessError, OSError, RuntimeError): - # see http://serverfault.com/questions/301128/how-to-download - if platform_is_win32(): - run_powershell([ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], - verbose=verbose, - exception=exception) - # Check if the RuntimeError raised by run(curl) should be silenced - elif verbose or exception: - raise - - -def verify(path, expected, verbose): - """Check if the sha256 sum of the given path is valid""" - if verbose: - eprint("verifying", path) - with open(path, "rb") as source: - found = hashlib.sha256(source.read()).hexdigest() - verified = found == expected - if not verified: - eprint("invalid checksum:\n" - " found: {}\n" - " expected: {}".format(found, expected)) - return verified - - -def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): - """Unpack the given tarball file""" - eprint("extracting", tarball) - fname = os.path.basename(tarball).replace(tarball_suffix, "") - with contextlib.closing(tarfile.open(tarball)) as tar: - for member in tar.getnames(): - if "/" not in member: - continue - name = member.replace(fname + "/", "", 1) - if match is not None and not name.startswith(match): - continue - name = name[len(match) + 1:] - - dst_path = os.path.join(dst, name) - if verbose: - eprint(" extracting", member) - tar.extract(member, dst) - src_path = os.path.join(dst, member) - if os.path.isdir(src_path) and os.path.exists(dst_path): - continue - shutil.move(src_path, dst_path) - shutil.rmtree(os.path.join(dst, fname)) - - -def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): - """Run a child program in a new process""" - if verbose: - eprint("running: " + ' '.join(args)) - sys.stdout.flush() - # Ensure that the .exe is used on Windows just in case a Linux ELF has been - # compiled in the same directory. - if os.name == 'nt' and not args[0].endswith('.exe'): - args[0] += '.exe' - # Use Popen here instead of call() as it apparently allows powershell on - # Windows to not lock up waiting for input presumably. - command_info = { - "command": args[0], - "args": args[1:], - "env": kwargs.get('env', os.environ.copy()), - "cwd": kwargs.get('cwd', os.getcwd()), - "type": "rust_compiler_invocation" - } - print(json.dumps(command_info)) - sys.exit(0) - -def run_powershell(script, *args, **kwargs): - """Run a powershell script""" - run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs) - - -def require(cmd, exit=True, exception=False): - '''Run a command, returning its output. - On error, - If `exception` is `True`, raise the error - Otherwise If `exit` is `True`, exit the process - Else return None.''' - try: - return subprocess.check_output(cmd).strip() - except (subprocess.CalledProcessError, OSError) as exc: - if exception: - raise - elif exit: - eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) - eprint("Please make sure it's installed and in the path.") - sys.exit(1) - return None - - - -def format_build_time(duration): - """Return a nicer format for build time - - >>> format_build_time('300') - '0:05:00' - """ - return str(datetime.timedelta(seconds=int(duration))) - - -def default_build_triple(verbose): - """Build triple as in LLVM""" - # If we're on Windows and have an existing `rustc` toolchain, use `rustc --version --verbose` - # to find our host target triple. This fixes an issue with Windows builds being detected - # as GNU instead of MSVC. - # Otherwise, detect it via `uname` - default_encoding = sys.getdefaultencoding() - - if platform_is_win32(): - try: - version = subprocess.check_output(["rustc", "--version", "--verbose"], - stderr=subprocess.DEVNULL) - version = version.decode(default_encoding) - host = next(x for x in version.split('\n') if x.startswith("host: ")) - triple = host.split("host: ")[1] - if verbose: - eprint("detected default triple {} from pre-installed rustc".format(triple)) - return triple - except Exception as e: - if verbose: - eprint("pre-installed rustc not detected: {}".format(e)) - eprint("falling back to auto-detect") - - required = not platform_is_win32() - uname = require(["uname", "-smp"], exit=required) - - # If we do not have `uname`, assume Windows. - if uname is None: - return 'x86_64-pc-windows-msvc' - - kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2) - - # The goal here is to come up with the same triple as LLVM would, - # at least for the subset of platforms we're willing to target. - kerneltype_mapper = { - 'Darwin': 'apple-darwin', - 'DragonFly': 'unknown-dragonfly', - 'FreeBSD': 'unknown-freebsd', - 'Haiku': 'unknown-haiku', - 'NetBSD': 'unknown-netbsd', - 'OpenBSD': 'unknown-openbsd', - 'GNU': 'unknown-hurd', - } - - # Consider the direct transformation first and then the special cases - if kernel in kerneltype_mapper: - kernel = kerneltype_mapper[kernel] - elif kernel == 'Linux': - # Apple doesn't support `-o` so this can't be used in the combined - # uname invocation above - ostype = require(["uname", "-o"], exit=required).decode(default_encoding) - if ostype == 'Android': - kernel = 'linux-android' - else: - kernel = 'unknown-linux-gnu' - elif kernel == 'SunOS': - kernel = 'pc-solaris' - # On Solaris, uname -m will return a machine classification instead - # of a cpu type, so uname -p is recommended instead. However, the - # output from that option is too generic for our purposes (it will - # always emit 'i386' on x86/amd64 systems). As such, isainfo -k - # must be used instead. - cputype = require(['isainfo', '-k']).decode(default_encoding) - # sparc cpus have sun as a target vendor - if 'sparc' in cputype: - kernel = 'sun-solaris' - elif kernel.startswith('MINGW'): - # msys' `uname` does not print gcc configuration, but prints msys - # configuration. so we cannot believe `uname -m`: - # msys1 is always i686 and msys2 is always x86_64. - # instead, msys defines $MSYSTEM which is MINGW32 on i686 and - # MINGW64 on x86_64. - kernel = 'pc-windows-gnu' - cputype = 'i686' - if os.environ.get('MSYSTEM') == 'MINGW64': - cputype = 'x86_64' - elif kernel.startswith('MSYS'): - kernel = 'pc-windows-gnu' - elif kernel.startswith('CYGWIN_NT'): - cputype = 'i686' - if kernel.endswith('WOW64'): - cputype = 'x86_64' - kernel = 'pc-windows-gnu' - elif platform_is_win32(): - # Some Windows platforms might have a `uname` command that returns a - # non-standard string (e.g. gnuwin32 tools returns `windows32`). In - # these cases, fall back to using sys.platform. - return 'x86_64-pc-windows-msvc' - elif kernel == 'AIX': - # `uname -m` returns the machine ID rather than machine hardware on AIX, - # so we are unable to use cputype to form triple. AIX 7.2 and - # above supports 32-bit and 64-bit mode simultaneously and `uname -p` - # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in - # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper - # are not used to infer AIX's triple. - return 'powerpc64-ibm-aix' - else: - err = "unknown OS type: {}".format(kernel) - sys.exit(err) - - if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd': - cputype = subprocess.check_output( - ['uname', '-p']).strip().decode(default_encoding) - cputype_mapper = { - 'BePC': 'i686', - 'aarch64': 'aarch64', - 'aarch64eb': 'aarch64', - 'amd64': 'x86_64', - 'arm64': 'aarch64', - 'i386': 'i686', - 'i486': 'i686', - 'i686': 'i686', - 'i686-AT386': 'i686', - 'i786': 'i686', - 'loongarch64': 'loongarch64', - 'm68k': 'm68k', - 'csky': 'csky', - 'powerpc': 'powerpc', - 'powerpc64': 'powerpc64', - 'powerpc64le': 'powerpc64le', - 'ppc': 'powerpc', - 'ppc64': 'powerpc64', - 'ppc64le': 'powerpc64le', - 'riscv64': 'riscv64gc', - 's390x': 's390x', - 'x64': 'x86_64', - 'x86': 'i686', - 'x86-64': 'x86_64', - 'x86_64': 'x86_64' - } - - # Consider the direct transformation first and then the special cases - if cputype in cputype_mapper: - cputype = cputype_mapper[cputype] - elif cputype in {'xscale', 'arm'}: - cputype = 'arm' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - elif kernel == 'unknown-freebsd': - cputype = processor - kernel = 'unknown-freebsd' - elif cputype == 'armv6l': - cputype = 'arm' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - else: - kernel += 'eabihf' - elif cputype in {'armv7l', 'armv8l'}: - cputype = 'armv7' - if kernel == 'linux-android': - kernel = 'linux-androideabi' - else: - kernel += 'eabihf' - elif cputype == 'mips': - if sys.byteorder == 'big': - cputype = 'mips' - elif sys.byteorder == 'little': - cputype = 'mipsel' - else: - raise ValueError("unknown byteorder: {}".format(sys.byteorder)) - elif cputype == 'mips64': - if sys.byteorder == 'big': - cputype = 'mips64' - elif sys.byteorder == 'little': - cputype = 'mips64el' - else: - raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) - # only the n64 ABI is supported, indicate it - kernel += 'abi64' - elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': - pass - else: - err = "unknown cpu type: {}".format(cputype) - sys.exit(err) - - return "{}-{}".format(cputype, kernel) - - -@contextlib.contextmanager -def output(filepath): - tmp = filepath + '.tmp' - with open(tmp, 'w') as f: - yield f - try: - if os.path.exists(filepath): - os.remove(filepath) # PermissionError/OSError on Win32 if in use - except OSError: - shutil.copy2(tmp, filepath) - os.remove(tmp) - return - os.rename(tmp, filepath) - - -class Stage0Toolchain: - def __init__(self, date, version): - self.date = date - self.version = version - - def channel(self): - return self.version + "-" + self.date - - -class DownloadInfo: - """A helper class that can be pickled into a parallel subprocess""" - - def __init__( - self, - base_download_url, - download_path, - bin_root, - tarball_path, - tarball_suffix, - stage0_data, - pattern, - verbose, - ): - self.base_download_url = base_download_url - self.download_path = download_path - self.bin_root = bin_root - self.tarball_path = tarball_path - self.tarball_suffix = tarball_suffix - self.stage0_data = stage0_data - self.pattern = pattern - self.verbose = verbose - -def download_component(download_info): - if not os.path.exists(download_info.tarball_path): - get( - download_info.base_download_url, - download_info.download_path, - download_info.tarball_path, - download_info.stage0_data, - verbose=download_info.verbose, - ) - -def unpack_component(download_info): - unpack( - download_info.tarball_path, - download_info.tarball_suffix, - download_info.bin_root, - match=download_info.pattern, - verbose=download_info.verbose, - ) - -class FakeArgs: - """Used for unit tests to avoid updating all call sites""" - def __init__(self): - self.build = '' - self.build_dir = '' - self.clean = False - self.verbose = False - self.json_output = False - self.color = 'auto' - self.warnings = 'default' - -class RustBuild(object): - """Provide all the methods required to build Rust""" - def __init__(self, config_toml="", args=None): - if args is None: - args = FakeArgs() - self.git_version = None - self.nix_deps_dir = None - self._should_fix_bins_and_dylibs = None - self.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) - - self.config_toml = config_toml - - self.clean = args.clean - self.json_output = args.json_output - self.verbose = args.verbose - self.color = args.color - self.warnings = args.warnings - - config_verbose_count = self.get_toml('verbose', 'build') - if config_verbose_count is not None: - self.verbose = max(self.verbose, int(config_verbose_count)) - - self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true' - self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true' - - build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build' - self.build_dir = os.path.abspath(build_dir) - - self.stage0_data = parse_stage0_file(os.path.join(self.rust_root, "src", "stage0")) - self.stage0_compiler = Stage0Toolchain( - self.stage0_data["compiler_date"], - self.stage0_data["compiler_version"] - ) - self.download_url = os.getenv("RUSTUP_DIST_SERVER") or self.stage0_data["dist_server"] - - self.build = args.build or self.build_triple() - - - def download_toolchain(self): - """Fetch the build system for Rust, written in Rust - - This method will build a cache directory, then it will fetch the - tarball which has the stage0 compiler used to then bootstrap the Rust - compiler itself. - - Each downloaded tarball is extracted, after that, the script - will move all the content to the right place. - """ - # Check if Nix-provided rustc and cargo exist - nix_rustc_exists = os.path.exists(self.rustc()) - nix_cargo_exists = os.path.exists(self.cargo()) - - # If Nix-provided tools exist, we don't need to download - need_rustc = not nix_rustc_exists - need_cargo = not nix_cargo_exists - - if need_rustc or need_cargo: - eprint("INFO: Nix-provided rustc or cargo not found. Proceeding with downloads.") - # For now, we will raise an exception if downloads are attempted - raise Exception("Downloads are disabled in Nix environment. Ensure rustc and cargo are provided via config.toml.") - else: - eprint("INFO: Nix-provided rustc and cargo found. Skipping downloads.") - return # Skip the rest of the download_toolchain method - - rustc_channel = self.stage0_compiler.version - bin_root = self.bin_root() - - key = self.stage0_compiler.date - is_outdated = self.program_out_of_date(self.rustc_stamp(), key) - need_rustc = self.rustc().startswith(bin_root) and (not os.path.exists(self.rustc()) \ - or is_outdated) - need_cargo = self.cargo().startswith(bin_root) and (not os.path.exists(self.cargo()) \ - or is_outdated) - - if need_rustc or need_cargo: - if os.path.exists(bin_root): - # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's - # running. Kill it. - if platform_is_win32(): - print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain") - regex = '{}\\\\(host|{})\\\\stage0\\\\libexec'.format( - os.path.basename(self.build_dir), - self.build - ) - script = ( - # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if - # the server isn't running. - 'Get-Process | ' + - 'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' + - 'Where-Object {{$_.Path -match "{}"}} |'.format(regex) + - 'Stop-Process' - ) - run_powershell([script]) - shutil.rmtree(bin_root) - - cache_dst = (self.get_toml('bootstrap-cache-path', 'build') or - os.path.join(self.build_dir, "cache")) - - rustc_cache = os.path.join(cache_dst, key) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) - - tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' - - toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - - tarballs_to_download = [] - - if need_rustc: - tarballs_to_download.append( - ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)) - ) - tarballs_to_download.append(("rustc-{}".format(toolchain_suffix), "rustc")) - - if need_cargo: - tarballs_to_download.append(("cargo-{}".format(toolchain_suffix), "cargo")) - - tarballs_download_info = [ - DownloadInfo( - base_download_url=self.download_url, - download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), - bin_root=self.bin_root(), - tarball_path=os.path.join(rustc_cache, filename), - tarball_suffix=tarball_suffix, - stage0_data=self.stage0_data, - pattern=pattern, - verbose=self.verbose, - ) - for filename, pattern in tarballs_to_download - ] - - # Download the components serially to show the progress bars properly. - for download_info in tarballs_download_info: - download_component(download_info) - - # Unpack the tarballs in parallle. - # In Python 2.7, Pool cannot be used as a context manager. - pool_size = min(len(tarballs_download_info), get_cpus()) - if self.verbose: - print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') - p = Pool(pool_size) - try: - # FIXME: A cheap workaround for https://github.com/rust-lang/rust/issues/125578, - # remove this once the issue is closed. - bootstrap_build_artifacts = os.path.join(self.bootstrap_out(), "debug") - if os.path.exists(bootstrap_build_artifacts): - shutil.rmtree(bootstrap_build_artifacts) - - p.map(unpack_component, tarballs_download_info) - finally: - p.close() - p.join() - - if self.should_fix_bins_and_dylibs(): - self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) - - self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) - self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) - self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root)) - lib_dir = "{}/lib".format(bin_root) - rustlib_bin_dir = "{}/rustlib/{}/bin".format(lib_dir, self.build) - self.fix_bin_or_dylib("{}/rust-lld".format(rustlib_bin_dir)) - self.fix_bin_or_dylib("{}/gcc-ld/ld.lld".format(rustlib_bin_dir)) - for lib in os.listdir(lib_dir): - # .so is not necessarily the suffix, there can be version numbers afterwards. - if ".so" in lib: - elf_path = os.path.join(lib_dir, lib) - with open(elf_path, "rb") as f: - magic = f.read(4) - # Patchelf will skip non-ELF files, but issue a warning. - if magic == b"\x7fELF": - self.fix_bin_or_dylib(elf_path) - - with output(self.rustc_stamp()) as rust_stamp: - rust_stamp.write(key) - - def should_fix_bins_and_dylibs(self): - """Whether or not `fix_bin_or_dylib` needs to be run; can only be True - on NixOS or if config.toml has `build.patch-binaries-for-nix` set. - """ - if self._should_fix_bins_and_dylibs is not None: - return self._should_fix_bins_and_dylibs - - def get_answer(): - default_encoding = sys.getdefaultencoding() - try: - ostype = subprocess.check_output( - ['uname', '-s']).strip().decode(default_encoding) - except subprocess.CalledProcessError: - return False - except OSError as reason: - if getattr(reason, 'winerror', None) is not None: - return False - raise reason - - if ostype != "Linux": - return False - - # If the user has explicitly indicated whether binaries should be - # patched for Nix, then don't check for NixOS. - if self.get_toml("patch-binaries-for-nix", "build") == "true": - return True - if self.get_toml("patch-binaries-for-nix", "build") == "false": - return False - - # Use `/etc/os-release` instead of `/etc/NIXOS`. - # The latter one does not exist on NixOS when using tmpfs as root. - try: - with open("/etc/os-release", "r") as f: - is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') - for ln in f) - except FileNotFoundError: - is_nixos = False - - # If not on NixOS, then warn if user seems to be atop Nix shell - if not is_nixos: - in_nix_shell = os.getenv('IN_NIX_SHELL') - if in_nix_shell: - eprint("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell), - "you may need to set `patch-binaries-for-nix=true` in config.toml") - - return is_nixos - - answer = self._should_fix_bins_and_dylibs = get_answer() - if answer: - eprint("INFO: You seem to be using Nix.") - return answer - - def fix_bin_or_dylib(self, fname): - """Modifies the interpreter section of 'fname' to fix the dynamic linker, - or the RPATH section, to fix the dynamic library search path - - This method is only required on NixOS and uses the PatchELF utility to - change the interpreter/RPATH of ELF executables. - - Please see https://nixos.org/patchelf.html for more information - """ - assert self._should_fix_bins_and_dylibs is True - eprint("attempting to patch", fname) - - # Only build `.nix-deps` once. - nix_deps_dir = self.nix_deps_dir - if not nix_deps_dir: - # Run `nix-build` to "build" each dependency (which will likely reuse - # the existing `/nix/store` copy, or at most download a pre-built copy). - # - # Importantly, we create a gc-root called `.nix-deps` in the `build/` - # directory, but still reference the actual `/nix/store` path in the rpath - # as it makes it significantly more robust against changes to the location of - # the `.nix-deps` location. - # - # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - # zlib: Needed as a system dependency of `libLLVM-*.so`. - # patchelf: Needed for patching ELF binaries (see doc comment above). - nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") - nix_expr = ''' - with (import {}); - symlinkJoin { - name = "rust-stage0-dependencies"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - ''' - try: - subprocess.check_output([ - "nix-build", "-E", nix_expr, "-o", nix_deps_dir, - ]) - except subprocess.CalledProcessError as reason: - eprint("WARNING: failed to call nix-build:", reason) - return - self.nix_deps_dir = nix_deps_dir - - patchelf = "{}/bin/patchelf".format(nix_deps_dir) - rpath_entries = [ - os.path.join(os.path.realpath(nix_deps_dir), "lib") - ] - patchelf_args = ["--add-rpath", ":".join(rpath_entries)] - if ".so" not in fname: - # Finally, set the correct .interp for binaries - with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: - patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] - - try: - subprocess.check_output([patchelf] + patchelf_args + [fname]) - except subprocess.CalledProcessError as reason: - eprint("WARNING: failed to call patchelf:", reason) - return - - def rustc_stamp(self): - """Return the path for .rustc-stamp at the given stage - - >>> rb = RustBuild() - >>> rb.build = "host" - >>> rb.build_dir = "build" - >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp") - >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp() - """ - return os.path.join(self.bin_root(), '.rustc-stamp') - - def program_out_of_date(self, stamp_path, key): - """Check if the given program stamp is out of date""" - if not os.path.exists(stamp_path) or self.clean: - return True - with open(stamp_path, 'r') as stamp: - return key != stamp.read() - - def bin_root(self): - """Return the binary root directory for the given stage - - >>> rb = RustBuild() - >>> rb.build = "devel" - >>> expected = os.path.abspath(os.path.join("build", "devel", "stage0")) - >>> assert rb.bin_root() == expected, rb.bin_root() - """ - subdir = "stage0" - return os.path.join(self.build_dir, self.build, subdir) - - def get_toml(self, key, section=None): - """Returns the value of the given key in config.toml, otherwise returns None - - >>> rb = RustBuild() - >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' - >>> rb.get_toml("key2") - 'value2' - - If the key does not exist, the result is None: - - >>> rb.get_toml("key3") is None - True - - Optionally also matches the section the key appears in - - >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' - >>> rb.get_toml('key', 'a') - 'value1' - >>> rb.get_toml('key', 'b') - 'value2' - >>> rb.get_toml('key', 'c') is None - True - - >>> rb.config_toml = 'key1 = true' - >>> rb.get_toml("key1") - 'true' - """ - return RustBuild.get_toml_static(self.config_toml, key, section) - - @staticmethod - def get_toml_static(config_toml, key, section=None): - cur_section = None - for line in config_toml.splitlines(): - section_match = re.match(r'^\s*\[(.*)\]\s*$', line) - if section_match is not None: - cur_section = section_match.group(1) - - match = re.match(r'^{}\s*=(.*)$'.format(key), line) - if match is not None: - value = match.group(1) - if section is None or section == cur_section: - return RustBuild.get_string(value) or value.strip() - return None - - def cargo(self): - """Return config path for cargo""" - return self.program_config('cargo') - - def rustc(self): - """Return config path for rustc""" - return self.program_config('rustc') - - def program_config(self, program): - """Return config path for the given program at the given stage - - >>> rb = RustBuild() - >>> rb.config_toml = 'rustc = "rustc"\\n' - >>> rb.program_config('rustc') - 'rustc' - >>> rb.config_toml = '' - >>> cargo_path = rb.program_config('cargo') - >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), - ... "bin", "cargo") - True - """ - config = self.get_toml(program) - if config: - return os.path.expanduser(config) - return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX)) - - @staticmethod - def get_string(line): - """Return the value between double quotes - - >>> RustBuild.get_string(' "devel" ') - 'devel' - >>> RustBuild.get_string(" 'devel' ") - 'devel' - >>> RustBuild.get_string('devel') is None - True - >>> RustBuild.get_string(' "devel ') - '' - """ - start = line.find('"') - if start != -1: - end = start + 1 + line[start + 1:].find('"') - return line[start + 1:end] - start = line.find('\'') - if start != -1: - end = start + 1 + line[start + 1:].find('\'') - return line[start + 1:end] - return None - - def bootstrap_out(self): - """Return the path of the bootstrap build artifacts - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap") - True - """ - return os.path.join(self.build_dir, "bootstrap") - - def bootstrap_binary(self): - """Return the path of the bootstrap binary - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", - ... "debug", "bootstrap") - True - """ - return os.path.join(self.bootstrap_out(), "debug", "bootstrap") - - def build_bootstrap(self): - """Build bootstrap""" - env = os.environ.copy() - if "GITHUB_ACTIONS" in env: - print("::group::Building bootstrap") - else: - eprint("Building bootstrap") - - args = self.build_bootstrap_cmd(env) - # Run this from the source directory so cargo finds .cargo/config - run(args, env=env, verbose=self.verbose, cwd=self.rust_root) - - if "GITHUB_ACTIONS" in env: - print("::endgroup::") - - def build_bootstrap_cmd(self, env): - """For tests.""" - build_dir = os.path.join(self.build_dir, "bootstrap") - if self.clean and os.path.exists(build_dir): - shutil.rmtree(build_dir) - # `CARGO_BUILD_TARGET` breaks bootstrap build. - # See also: . - if "CARGO_BUILD_TARGET" in env: - del env["CARGO_BUILD_TARGET"] - env["CARGO_TARGET_DIR"] = build_dir - env["RUSTC"] = self.rustc() - env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LD_LIBRARY_PATH"]) \ - if "LD_LIBRARY_PATH" in env else "" - env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ - if "DYLD_LIBRARY_PATH" in env else "" - env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LIBRARY_PATH"]) \ - if "LIBRARY_PATH" in env else "" - env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LIBPATH"]) \ - if "LIBPATH" in env else "" - - # Export Stage0 snapshot compiler related env variables - build_section = "target.{}".format(self.build) - host_triple_sanitized = self.build.replace("-", "_") - var_data = { - "CC": "cc", "CXX": "cxx", "LD": "linker", "AR": "ar", "RANLIB": "ranlib" - } - for var_name, toml_key in var_data.items(): - toml_val = self.get_toml(toml_key, build_section) - if toml_val is not None: - env["{}_{}".format(var_name, host_triple_sanitized)] = toml_val - - # In src/etc/rust_analyzer_settings.json, we configure rust-analyzer to - # pass RUSTC_BOOTSTRAP=1 to all cargo invocations because the standard - # library uses unstable Cargo features. Without RUSTC_BOOTSTRAP, - # rust-analyzer would fail to fetch workspace layout when the system's - # default toolchain is not nightly. - # - # But that setting has the collateral effect of rust-analyzer also - # passing RUSTC_BOOTSTRAP=1 to all x.py invocations too (the various - # overrideCommand). - # - # Set a consistent RUSTC_BOOTSTRAP=1 here to prevent spurious rebuilds - # of bootstrap when rust-analyzer x.py invocations are interleaved with - # handwritten ones on the command line. - env["RUSTC_BOOTSTRAP"] = "1" - - # If any of RUSTFLAGS or RUSTFLAGS_BOOTSTRAP are present and nonempty, - # we allow arbitrary compiler flags in there, including unstable ones - # such as `-Zthreads=8`. - # - # But if there aren't custom flags being passed to bootstrap, then we - # cancel the RUSTC_BOOTSTRAP=1 from above by passing `-Zallow-features=` - # to ensure unstable language or library features do not accidentally - # get introduced into bootstrap over time. Distros rely on being able to - # compile bootstrap with a variety of their toolchains, not necessarily - # the same as Rust's CI uses. - if env.get("RUSTFLAGS", "") or env.get("RUSTFLAGS_BOOTSTRAP", ""): - # Preserve existing RUSTFLAGS. - env.setdefault("RUSTFLAGS", "") - else: - env["RUSTFLAGS"] = "-Zallow-features=" - - target_features = [] - if self.get_toml("crt-static", build_section) == "true": - target_features += ["+crt-static"] - elif self.get_toml("crt-static", build_section) == "false": - target_features += ["-crt-static"] - if target_features: - env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) - target_linker = self.get_toml("linker", build_section) - if target_linker is not None: - env["RUSTFLAGS"] += " -C linker=" + target_linker - # When changing this list, also update the corresponding list in `Builder::cargo` - # in `src/bootstrap/src/core/builder.rs`. - env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" - if self.warnings == "default": - deny_warnings = self.get_toml("deny-warnings", "rust") != "false" - else: - deny_warnings = self.warnings == "deny" - if deny_warnings: - env["RUSTFLAGS"] += " -Dwarnings" - - # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. - # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of - # RUSTFLAGS to be actually effective (e.g., if we have `-Dwarnings` in - # RUSTFLAGS, passing `-Awarnings` from RUSTFLAGS_BOOTSTRAP should override it). - if "RUSTFLAGS_BOOTSTRAP" in env: - env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] - - env["PATH"] = os.path.join(self.bin_root(), "bin") + \ - os.pathsep + env["PATH"] - if not os.path.isfile(self.cargo()): - raise Exception("no cargo executable found at `{}`".format( - self.cargo())) - args = [self.cargo(), "build", "--manifest-path", - os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] - args.extend("--verbose" for _ in range(self.verbose)) - if self.use_locked_deps: - args.append("--locked") - if self.use_vendored_sources: - args.append("--frozen") - if self.get_toml("metrics", "build"): - args.append("--features") - args.append("build-metrics") - if self.json_output: - args.append("--message-format=json") - if self.color == "always": - args.append("--color=always") - elif self.color == "never": - args.append("--color=never") - try: - args += env["CARGOFLAGS"].split() - except KeyError: - pass - - return args - - def build_triple(self): - """Build triple as in LLVM - - Note that `default_build_triple` is moderately expensive, - so use `self.build` where possible. - """ - config = self.get_toml('build') - return config or default_build_triple(self.verbose) - - def check_vendored_status(self): - """Check that vendoring is configured properly""" - # keep this consistent with the equivalent check in bootstrap: - # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 - if 'SUDO_USER' in os.environ and not self.use_vendored_sources: - if os.getuid() == 0: - self.use_vendored_sources = True - eprint('INFO: looks like you\'re trying to run this command as root') - eprint(' and so in order to preserve your $HOME this will now') - eprint(' use vendored sources by default.') - - cargo_dir = os.path.join(self.rust_root, '.cargo') - if self.use_vendored_sources: - vendor_dir = os.path.join(self.rust_root, 'vendor') - if not os.path.exists(vendor_dir): - eprint('ERROR: vendoring required, but vendor directory does not exist.') - eprint(' Run `x.py vendor` to initialize the vendor directory.') - eprint(' Alternatively, use the pre-vendored `rustc-src` dist component.') - eprint(' To get a stable/beta/nightly version, download it from: ') - eprint(' ' - 'https://forge.rust-lang.org/infra/other-installation-methods.html#source-code') - eprint(' To get a specific commit version, download it using the below URL,') - eprint(' replacing with a specific commit checksum: ') - eprint(' ' - 'https://ci-artifacts.rust-lang.org/rustc-builds//rustc-nightly-src.tar.xz') - eprint(' Once you have the source downloaded, place the vendor directory') - eprint(' from the archive in the root of the rust project.') - raise Exception("{} not found".format(vendor_dir)) - - if not os.path.exists(cargo_dir): - eprint('ERROR: vendoring required, but .cargo/config does not exist.') - raise Exception("{} not found".format(cargo_dir)) - -def parse_args(args): - """Parse the command line arguments that the python script needs.""" - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('-h', '--help', action='store_true') - parser.add_argument('--config') - parser.add_argument('--build-dir') - parser.add_argument('--build') - parser.add_argument('--color', choices=['always', 'never', 'auto']) - parser.add_argument('--clean', action='store_true') - parser.add_argument('--json-output', action='store_true') - parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default') - parser.add_argument('-v', '--verbose', action='count', default=0) - - return parser.parse_known_args(args)[0] - -def parse_stage0_file(path): - # Check for RUST_SRC_STAGE0_PATH environment variable - stage0_path_from_env = os.environ.get("RUST_SRC_STAGE0_PATH") - if stage0_path_from_env: - path = stage0_path_from_env - eprint(f"INFO: Using stage0 file from RUST_SRC_STAGE0_PATH: {path}") - - result = {} - with open(path, 'r') as file: - for line in file: - line = line.strip() - if line and not line.startswith('#'): - key, value = line.split('=', 1) - result[key.strip()] = value.strip() - return result - -def bootstrap(args): - """Configure, fetch, build and run the initial bootstrap""" - rust_root = os.path.abspath(os.path.join(__file__, '../../..')) - - if not os.path.exists(os.path.join(rust_root, '.git')) and \ - os.path.exists(os.path.join(rust_root, '.github')): - eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " - "git clone nor distributed tarball.\nThis build may fail due to missing submodules " - "unless you put them in place manually.") - - # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, - # then `config.toml` in the root directory. - toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') - using_default_path = toml_path is None - if using_default_path: - toml_path = 'config.toml' - if not os.path.exists(toml_path): - toml_path = os.path.join(rust_root, toml_path) - - # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - # but not if `config.toml` hasn't been created. - if not using_default_path or os.path.exists(toml_path): - with open(toml_path) as config: - config_toml = config.read() - else: - config_toml = '' - - profile = RustBuild.get_toml_static(config_toml, "profile") - is_non_git_source = not os.path.exists(os.path.join(rust_root, ".git")) - - if profile is None and is_non_git_source: - profile = "dist" - - if profile is not None: - # Allows creating alias for profile names, allowing - # profiles to be renamed while maintaining back compatibility - # Keep in sync with `profile_aliases` in config.rs - profile_aliases = { - "user": "dist" - } - include_file = 'config.{}.toml'.format(profile_aliases.get(profile) or profile) - include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults') - include_path = os.path.join(include_dir, include_file) - - if not os.path.exists(include_path): - raise Exception("Unrecognized config profile '{}'. Check src/bootstrap/defaults" - " for available options.".format(profile)) - - # HACK: This works because `self.get_toml()` returns the first match it finds for a - # specific key, so appending our defaults at the end allows the user to override them - with open(include_path) as included_toml: - config_toml += os.linesep + included_toml.read() - - # Configure initial bootstrap - build = RustBuild(config_toml, args) - build.check_vendored_status() - - if not os.path.exists(build.build_dir): - os.makedirs(build.build_dir) - - # Fetch/build the bootstrap - build.download_toolchain() - sys.stdout.flush() - build.build_bootstrap() - sys.stdout.flush() - - # Run the bootstrap - args = [build.bootstrap_binary()] - args.extend(sys.argv[1:]) - env = os.environ.copy() - env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) - env["BOOTSTRAP_PYTHON"] = sys.executable - run(args, env=env, verbose=build.verbose, is_bootstrap=True) - - -def main(): - """Entry point for the bootstrap process""" - start_time = time() - - # x.py help ... - if len(sys.argv) > 1 and sys.argv[1] == 'help': - sys.argv[1] = '-h' - - args = parse_args(sys.argv) - help_triggered = args.help or len(sys.argv) == 1 - - # If the user is asking for help, let them know that the whole download-and-build - # process has to happen before anything is printed out. - if help_triggered: - eprint( - "INFO: Downloading and building bootstrap before processing --help command.\n" - " See src/bootstrap/README.md for help with common commands.") - - exit_code = 0 - success_word = "successfully" - try: - bootstrap(args) - except (SystemExit, KeyboardInterrupt) as error: - if hasattr(error, 'code') and isinstance(error.code, int): - exit_code = error.code - else: - exit_code = 1 - eprint(error) - success_word = "unsuccessfully" - - if not help_triggered: - eprint("Build completed", success_word, "in", format_build_time(time() - start_time)) - sys.exit(exit_code) - - -if __name__ == '__main__': - main() diff --git a/standalonex/src/bootstrap/bootstrap_test.py b/standalonex/src/bootstrap/bootstrap_test.py deleted file mode 100644 index 70ed12b9..00000000 --- a/standalonex/src/bootstrap/bootstrap_test.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Bootstrap tests - -Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstrap_test.py`.""" - -from __future__ import absolute_import, division, print_function -import os -import unittest -from unittest.mock import patch -import tempfile -import hashlib -import sys - -from shutil import rmtree - -# Allow running this from the top-level directory. -bootstrap_dir = os.path.dirname(os.path.abspath(__file__)) -# For the import below, have Python search in src/bootstrap first. -sys.path.insert(0, bootstrap_dir) -import bootstrap # noqa: E402 -import configure # noqa: E402 - -def serialize_and_parse(configure_args, bootstrap_args=None): - from io import StringIO - - if bootstrap_args is None: - bootstrap_args = bootstrap.FakeArgs() - - section_order, sections, targets = configure.parse_args(configure_args) - buffer = StringIO() - configure.write_config_toml(buffer, section_order, targets, sections) - build = bootstrap.RustBuild(config_toml=buffer.getvalue(), args=bootstrap_args) - - try: - import tomllib - # Verify this is actually valid TOML. - tomllib.loads(build.config_toml) - except ImportError: - print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) - return build - - -class VerifyTestCase(unittest.TestCase): - """Test Case for verify""" - def setUp(self): - self.container = tempfile.mkdtemp() - self.src = os.path.join(self.container, "src.txt") - self.bad_src = os.path.join(self.container, "bad.txt") - content = "Hello world" - - self.expected = hashlib.sha256(content.encode("utf-8")).hexdigest() - - with open(self.src, "w") as src: - src.write(content) - with open(self.bad_src, "w") as bad: - bad.write("Hello!") - - def tearDown(self): - rmtree(self.container) - - def test_valid_file(self): - """Check if the sha256 sum of the given file is valid""" - self.assertTrue(bootstrap.verify(self.src, self.expected, False)) - - def test_invalid_file(self): - """Should verify that the file is invalid""" - self.assertFalse(bootstrap.verify(self.bad_src, self.expected, False)) - - -class ProgramOutOfDate(unittest.TestCase): - """Test if a program is out of date""" - def setUp(self): - self.container = tempfile.mkdtemp() - os.mkdir(os.path.join(self.container, "stage0")) - self.build = bootstrap.RustBuild() - self.build.date = "2017-06-15" - self.build.build_dir = self.container - self.rustc_stamp_path = os.path.join(self.container, "stage0", - ".rustc-stamp") - self.key = self.build.date + str(None) - - def tearDown(self): - rmtree(self.container) - - def test_stamp_path_does_not_exist(self): - """Return True when the stamp file does not exist""" - if os.path.exists(self.rustc_stamp_path): - os.unlink(self.rustc_stamp_path) - self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - def test_dates_are_different(self): - """Return True when the dates are different""" - with open(self.rustc_stamp_path, "w") as rustc_stamp: - rustc_stamp.write("2017-06-14None") - self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - def test_same_dates(self): - """Return False both dates match""" - with open(self.rustc_stamp_path, "w") as rustc_stamp: - rustc_stamp.write("2017-06-15None") - self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - -class ParseArgsInConfigure(unittest.TestCase): - """Test if `parse_args` function in `configure.py` works properly""" - @patch("configure.err") - def test_unknown_args(self, err): - # It should be print an error message if the argument doesn't start with '--' - configure.parse_args(["enable-full-tools"]) - err.assert_called_with("Option 'enable-full-tools' is not recognized") - err.reset_mock() - # It should be print an error message if the argument is not recognized - configure.parse_args(["--some-random-flag"]) - err.assert_called_with("Option '--some-random-flag' is not recognized") - - @patch("configure.err") - def test_need_value_args(self, err): - """It should print an error message if a required argument value is missing""" - configure.parse_args(["--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - - @patch("configure.err") - def test_option_checking(self, err): - # Options should be checked even if `--enable-option-checking` is not passed - configure.parse_args(["--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - err.reset_mock() - # Options should be checked if `--enable-option-checking` is passed - configure.parse_args(["--enable-option-checking", "--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - err.reset_mock() - # Options should not be checked if `--disable-option-checking` is passed - configure.parse_args(["--disable-option-checking", "--target"]) - err.assert_not_called() - - @patch("configure.parse_example_config", lambda known_args, _: known_args) - def test_known_args(self): - # It should contain known and correct arguments - known_args = configure.parse_args(["--enable-full-tools"]) - self.assertTrue(known_args["full-tools"][0][1]) - known_args = configure.parse_args(["--disable-full-tools"]) - self.assertFalse(known_args["full-tools"][0][1]) - # It should contain known arguments and their values - known_args = configure.parse_args(["--target=x86_64-unknown-linux-gnu"]) - self.assertEqual(known_args["target"][0][1], "x86_64-unknown-linux-gnu") - known_args = configure.parse_args(["--target", "x86_64-unknown-linux-gnu"]) - self.assertEqual(known_args["target"][0][1], "x86_64-unknown-linux-gnu") - - -class GenerateAndParseConfig(unittest.TestCase): - """Test that we can serialize and deserialize a config.toml file""" - def test_no_args(self): - build = serialize_and_parse([]) - self.assertEqual(build.get_toml("profile"), 'dist') - self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) - - def test_set_section(self): - build = serialize_and_parse(["--set", "llvm.download-ci-llvm"]) - self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), 'true') - - def test_set_target(self): - build = serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"]) - self.assertEqual(build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), 'gcc') - - def test_set_top_level(self): - build = serialize_and_parse(["--set", "profile=compiler"]) - self.assertEqual(build.get_toml("profile"), 'compiler') - - def test_set_codegen_backends(self): - build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift']"), -1) - build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift,llvm"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift', 'llvm']"), -1) - build = serialize_and_parse(["--enable-full-tools"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['llvm']"), -1) - - -class BuildBootstrap(unittest.TestCase): - """Test that we generate the appropriate arguments when building bootstrap""" - - def build_args(self, configure_args=None, args=None, env=None): - if configure_args is None: - configure_args = [] - if args is None: - args = [] - if env is None: - env = {} - - # This test ends up invoking build_bootstrap_cmd, which searches for - # the Cargo binary and errors out if it cannot be found. This is not a - # problem in most cases, but there is a scenario where it would cause - # the test to fail. - # - # When a custom local Cargo is configured in config.toml (with the - # build.cargo setting), no Cargo is downloaded to any location known by - # bootstrap, and bootstrap relies on that setting to find it. - # - # In this test though we are not using the config.toml of the caller: - # we are generating a blank one instead. If we don't set build.cargo in - # it, the test will have no way to find Cargo, failing the test. - cargo_bin = os.environ.get("BOOTSTRAP_TEST_CARGO_BIN") - if cargo_bin is not None: - configure_args += ["--set", "build.cargo=" + cargo_bin] - rustc_bin = os.environ.get("BOOTSTRAP_TEST_RUSTC_BIN") - if rustc_bin is not None: - configure_args += ["--set", "build.rustc=" + rustc_bin] - - env = env.copy() - env["PATH"] = os.environ["PATH"] - - parsed = bootstrap.parse_args(args) - build = serialize_and_parse(configure_args, parsed) - # Make these optional so that `python -m unittest` works when run manually. - build_dir = os.environ.get("BUILD_DIR") - if build_dir is not None: - build.build_dir = build_dir - build_platform = os.environ.get("BUILD_PLATFORM") - if build_platform is not None: - build.build = build_platform - return build.build_bootstrap_cmd(env), env - - def test_cargoflags(self): - args, _ = self.build_args(env={"CARGOFLAGS": "--timings"}) - self.assertTrue("--timings" in args) - - def test_warnings(self): - for toml_warnings in ['false', 'true', None]: - configure_args = [] - if toml_warnings is not None: - configure_args = ["--set", "rust.deny-warnings=" + toml_warnings] - - _, env = self.build_args(configure_args, args=["--warnings=warn"]) - self.assertFalse("-Dwarnings" in env["RUSTFLAGS"]) - - _, env = self.build_args(configure_args, args=["--warnings=deny"]) - self.assertTrue("-Dwarnings" in env["RUSTFLAGS"]) diff --git a/standalonex/src/bootstrap/configure.py b/standalonex/src/bootstrap/configure.py deleted file mode 100755 index 70f4e709..00000000 --- a/standalonex/src/bootstrap/configure.py +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env python - -# ignore-tidy-linelength - -from __future__ import absolute_import, division, print_function -import shlex -import sys -import os -rust_dir = os.path.dirname(os.path.abspath(__file__)) -rust_dir = os.path.dirname(rust_dir) -rust_dir = os.path.dirname(rust_dir) -sys.path.append(os.path.join(rust_dir, "src", "bootstrap")) -import bootstrap # noqa: E402 - - -class Option(object): - def __init__(self, name, rustbuild, desc, value): - self.name = name - self.rustbuild = rustbuild - self.desc = desc - self.value = value - - -options = [] - - -def o(*args): - options.append(Option(*args, value=False)) - - -def v(*args): - options.append(Option(*args, value=True)) - - -o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code") -o("docs", "build.docs", "build standard library documentation") -o("compiler-docs", "build.compiler-docs", "build compiler documentation") -o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") -o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests") -o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") -o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds") -o("local-rust", None, "use an installed rustc rather than downloading a snapshot") -v("local-rust-root", None, "set prefix for local rust binary") -o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version") -o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM") -o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") -o("rpath", "rust.rpath", "build rpaths into rustc itself") -o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests") -o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") -o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date") -o("vendor", "build.vendor", "enable usage of vendored Rust crates") -o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, dfsan, lsan, msan, tsan, hwasan)") -o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball") -o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo") -o("profiler", "build.profiler", "build the profiler runtime") -o("full-tools", None, "enable all tools") -o("lld", "rust.lld", "build lld") -o("llvm-bitcode-linker", "rust.llvm-bitcode-linker", "build llvm bitcode linker") -o("clang", "llvm.clang", "build clang") -o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") -o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") -o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") -o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") - -v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") -v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") -v("llvm-ldflags", "llvm.ldflags", "build LLVM with these extra linker flags") - -v("llvm-libunwind", "rust.llvm-libunwind", "use LLVM libunwind") - -# Optimization and debugging options. These may be overridden by the release -# channel, etc. -o("optimize-llvm", "llvm.optimize", "build optimized LLVM") -o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") -o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") -o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") -o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") -o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") -o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions") -o("overflow-checks", "rust.overflow-checks", "build with overflow checks") -o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks") -o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata") -v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code") -v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler") -v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library") -v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools") -v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest") -v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file") - -v("prefix", "install.prefix", "set installation prefix") -v("localstatedir", "install.localstatedir", "local state directory") -v("datadir", "install.datadir", "install data") -v("sysconfdir", "install.sysconfdir", "install system configuration files") -v("infodir", "install.infodir", "install additional info") -v("libdir", "install.libdir", "install libraries") -v("mandir", "install.mandir", "install man pages in PATH") -v("docdir", "install.docdir", "install documentation in PATH") -v("bindir", "install.bindir", "install binaries") - -v("llvm-root", None, "set LLVM root") -v("llvm-config", None, "set path to llvm-config") -v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") -v("python", "build.python", "set path to python") -v("android-ndk", "build.android-ndk", "set path to Android NDK") -v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", - "MUSL root installation directory (deprecated)") -v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", - "x86_64-unknown-linux-musl install directory") -v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root", - "i586-unknown-linux-musl install directory") -v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root", - "i686-unknown-linux-musl install directory") -v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root", - "arm-unknown-linux-musleabi install directory") -v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root", - "arm-unknown-linux-musleabihf install directory") -v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root", - "armv5te-unknown-linux-musleabi install directory") -v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root", - "armv7-unknown-linux-musleabi install directory") -v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root", - "armv7-unknown-linux-musleabihf install directory") -v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root", - "aarch64-unknown-linux-musl install directory") -v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root", - "mips-unknown-linux-musl install directory") -v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root", - "mipsel-unknown-linux-musl install directory") -v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root", - "mips64-unknown-linux-muslabi64 install directory") -v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root", - "mips64el-unknown-linux-muslabi64 install directory") -v("musl-root-riscv32gc", "target.riscv32gc-unknown-linux-musl.musl-root", - "riscv32gc-unknown-linux-musl install directory") -v("musl-root-riscv64gc", "target.riscv64gc-unknown-linux-musl.musl-root", - "riscv64gc-unknown-linux-musl install directory") -v("musl-root-loongarch64", "target.loongarch64-unknown-linux-musl.musl-root", - "loongarch64-unknown-linux-musl install directory") -v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("experimental-targets", "llvm.experimental-targets", - "experimental LLVM targets to build") -v("release-channel", "rust.channel", "the name of the release channel to build") -v("release-description", "rust.description", "optional descriptive string for version output") -v("dist-compression-formats", None, "List of compression formats to use") - -# Used on systems where "cc" is unavailable -v("default-linker", "rust.default-linker", "the default linker") - -# Many of these are saved below during the "writing configuration" step -# (others are conditionally saved). -o("manage-submodules", "build.submodules", "let the build manage the git submodules") -o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)") -o("extended", "build.extended", "build an extended rust tool set") - -v("bootstrap-cache-path", None, "use provided path for the bootstrap cache") -v("tools", None, "List of extended tools will be installed") -v("codegen-backends", None, "List of codegen backends to build") -v("build", "build.build", "GNUs ./configure syntax LLVM build triple") -v("host", None, "List of GNUs ./configure syntax LLVM host triples") -v("target", None, "List of GNUs ./configure syntax LLVM target triples") - -# Options specific to this configure script -o("option-checking", None, "complain about unrecognized options in this configure script") -o("verbose-configure", None, "don't truncate options when printing them in this configure script") -v("set", None, "set arbitrary key/value pairs in TOML configuration") - - -def p(msg): - print("configure: " + msg) - - -def err(msg): - print("\nconfigure: ERROR: " + msg + "\n") - sys.exit(1) - -def is_value_list(key): - for option in options: - if option.name == key and option.desc.startswith('List of'): - return True - return False - -if '--help' in sys.argv or '-h' in sys.argv: - print('Usage: ./configure [options]') - print('') - print('Options') - for option in options: - if 'android' in option.name: - # no one needs to know about these obscure options - continue - if option.value: - print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc)) - else: - print('\t--enable-{:25} OR --disable-{}'.format(option.name, option.name)) - print('\t\t' + option.desc) - print('') - print('This configure script is a thin configuration shim over the true') - print('configuration system, `config.toml`. You can explore the comments') - print('in `config.example.toml` next to this configure script to see') - print('more information about what each option is. Additionally you can') - print('pass `--set` as an argument to set arbitrary key/value pairs') - print('in the TOML configuration if desired') - print('') - print('Also note that all options which take `--enable` can similarly') - print('be passed with `--disable-foo` to forcibly disable the option') - sys.exit(0) - -VERBOSE = False - -# Parse all command line arguments into one of these three lists, handling -# boolean and value-based options separately -def parse_args(args): - unknown_args = [] - need_value_args = [] - known_args = {} - - i = 0 - while i < len(args): - arg = args[i] - i += 1 - if not arg.startswith('--'): - unknown_args.append(arg) - continue - - found = False - for option in options: - value = None - if option.value: - keyval = arg[2:].split('=', 1) - key = keyval[0] - if option.name != key: - continue - - if len(keyval) > 1: - value = keyval[1] - elif i < len(args): - value = args[i] - i += 1 - else: - need_value_args.append(arg) - continue - else: - if arg[2:] == 'enable-' + option.name: - value = True - elif arg[2:] == 'disable-' + option.name: - value = False - else: - continue - - found = True - if option.name not in known_args: - known_args[option.name] = [] - known_args[option.name].append((option, value)) - break - - if not found: - unknown_args.append(arg) - - # NOTE: here and a few other places, we use [-1] to apply the *last* value - # passed. But if option-checking is enabled, then the known_args loop will - # also assert that options are only passed once. - option_checking = ('option-checking' not in known_args - or known_args['option-checking'][-1][1]) - if option_checking: - if len(unknown_args) > 0: - err("Option '" + unknown_args[0] + "' is not recognized") - if len(need_value_args) > 0: - err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) - - global VERBOSE - VERBOSE = 'verbose-configure' in known_args - - config = {} - - set('build.configure-args', args, config) - apply_args(known_args, option_checking, config) - return parse_example_config(known_args, config) - - -def build(known_args): - if 'build' in known_args: - return known_args['build'][-1][1] - return bootstrap.default_build_triple(verbose=False) - - -def set(key, value, config): - if isinstance(value, list): - # Remove empty values, which value.split(',') tends to generate and - # replace single quotes for double quotes to ensure correct parsing. - value = [v.replace('\'', '"') for v in value if v] - - s = "{:20} := {}".format(key, value) - if len(s) < 70 or VERBOSE: - p(s) - else: - p(s[:70] + " ...") - - arr = config - - # Split `key` on periods using shell semantics. - lexer = shlex.shlex(key, posix=True) - lexer.whitespace = "." - lexer.wordchars += "-" - parts = list(lexer) - - for i, part in enumerate(parts): - if i == len(parts) - 1: - if is_value_list(part) and isinstance(value, str): - value = value.split(',') - arr[part] = value - else: - if part not in arr: - arr[part] = {} - arr = arr[part] - - -def apply_args(known_args, option_checking, config): - for key in known_args: - # The `set` option is special and can be passed a bunch of times - if key == 'set': - for _option, value in known_args[key]: - keyval = value.split('=', 1) - if len(keyval) == 1 or keyval[1] == "true": - value = True - elif keyval[1] == "false": - value = False - else: - value = keyval[1] - set(keyval[0], value, config) - continue - - # Ensure each option is only passed once - arr = known_args[key] - if option_checking and len(arr) > 1: - err("Option '{}' provided more than once".format(key)) - option, value = arr[-1] - - # If we have a clear avenue to set our value in rustbuild, do so - if option.rustbuild is not None: - set(option.rustbuild, value, config) - continue - - # Otherwise we're a "special" option and need some extra handling, so do - # that here. - build_triple = build(known_args) - - if option.name == 'sccache': - set('llvm.ccache', 'sccache', config) - elif option.name == 'local-rust': - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/rustc'): - set('build.rustc', path + '/rustc', config) - break - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/cargo'): - set('build.cargo', path + '/cargo', config) - break - elif option.name == 'local-rust-root': - set('build.rustc', value + '/bin/rustc', config) - set('build.cargo', value + '/bin/cargo', config) - elif option.name == 'llvm-root': - set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config) - elif option.name == 'llvm-config': - set('target.{}.llvm-config'.format(build_triple), value, config) - elif option.name == 'llvm-filecheck': - set('target.{}.llvm-filecheck'.format(build_triple), value, config) - elif option.name == 'tools': - set('build.tools', value.split(','), config) - elif option.name == 'bootstrap-cache-path': - set('build.bootstrap-cache-path', value, config) - elif option.name == 'codegen-backends': - set('rust.codegen-backends', value.split(','), config) - elif option.name == 'host': - set('build.host', value.split(','), config) - elif option.name == 'target': - set('build.target', value.split(','), config) - elif option.name == 'full-tools': - set('rust.codegen-backends', ['llvm'], config) - set('rust.lld', True, config) - set('rust.llvm-tools', True, config) - set('rust.llvm-bitcode-linker', True, config) - set('build.extended', True, config) - elif option.name in ['option-checking', 'verbose-configure']: - # this was handled above - pass - elif option.name == 'dist-compression-formats': - set('dist.compression-formats', value.split(','), config) - else: - raise RuntimeError("unhandled option {}".format(option.name)) - -# "Parse" the `config.example.toml` file into the various sections, and we'll -# use this as a template of a `config.toml` to write out which preserves -# all the various comments and whatnot. -# -# Note that the `target` section is handled separately as we'll duplicate it -# per configured target, so there's a bit of special handling for that here. -def parse_example_config(known_args, config): - sections = {} - cur_section = None - sections[None] = [] - section_order = [None] - targets = {} - top_level_keys = [] - - with open(rust_dir + '/config.example.toml') as example_config: - example_lines = example_config.read().split("\n") - for line in example_lines: - if cur_section is None: - if line.count('=') == 1: - top_level_key = line.split('=')[0] - top_level_key = top_level_key.strip(' #') - top_level_keys.append(top_level_key) - if line.startswith('['): - cur_section = line[1:-1] - if cur_section.startswith('target'): - cur_section = 'target' - elif '.' in cur_section: - raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) - sections[cur_section] = [line] - section_order.append(cur_section) - else: - sections[cur_section].append(line) - - # Fill out the `targets` array by giving all configured targets a copy of the - # `target` section we just loaded from the example config - configured_targets = [build(known_args)] - if 'build' in config: - if 'host' in config['build']: - configured_targets += config['build']['host'] - if 'target' in config['build']: - configured_targets += config['build']['target'] - if 'target' in config: - for target in config['target']: - configured_targets.append(target) - for target in configured_targets: - targets[target] = sections['target'][:] - # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. - # Avoid using quotes unless it's necessary. - targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) - - if 'profile' not in config: - set('profile', 'dist', config) - configure_file(sections, top_level_keys, targets, config) - return section_order, sections, targets - - -def is_number(value): - try: - float(value) - return True - except ValueError: - return False - - -# Here we walk through the constructed configuration we have from the parsed -# command line arguments. We then apply each piece of configuration by -# basically just doing a `sed` to change the various configuration line to what -# we've got configure. -def to_toml(value): - if isinstance(value, bool): - if value: - return "true" - else: - return "false" - elif isinstance(value, list): - return '[' + ', '.join(map(to_toml, value)) + ']' - elif isinstance(value, str): - # Don't put quotes around numeric values - if is_number(value): - return value - else: - return "'" + value + "'" - elif isinstance(value, dict): - return "{" + ", ".join(map(lambda a: "{} = {}".format(to_toml(a[0]), to_toml(a[1])), value.items())) + "}" - else: - raise RuntimeError('no toml') - - -def configure_section(lines, config): - for key in config: - value = config[key] - found = False - for i, line in enumerate(lines): - if not line.startswith('#' + key + ' = '): - continue - found = True - lines[i] = "{} = {}".format(key, to_toml(value)) - break - if not found: - # These are used by rpm, but aren't accepted by x.py. - # Give a warning that they're ignored, but not a hard error. - if key in ["infodir", "localstatedir"]: - print("WARNING: {} will be ignored".format(key)) - else: - raise RuntimeError("failed to find config line for {}".format(key)) - - -def configure_top_level_key(lines, top_level_key, value): - for i, line in enumerate(lines): - if line.startswith('#' + top_level_key + ' = ') or line.startswith(top_level_key + ' = '): - lines[i] = "{} = {}".format(top_level_key, to_toml(value)) - return - - raise RuntimeError("failed to find config line for {}".format(top_level_key)) - - -# Modify `sections` to reflect the parsed arguments and example configs. -def configure_file(sections, top_level_keys, targets, config): - for section_key, section_config in config.items(): - if section_key not in sections and section_key not in top_level_keys: - raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) - if section_key in top_level_keys: - configure_top_level_key(sections[None], section_key, section_config) - - elif section_key == 'target': - for target in section_config: - configure_section(targets[target], section_config[target]) - else: - configure_section(sections[section_key], section_config) - - -def write_uncommented(target, f): - block = [] - is_comment = True - - for line in target: - block.append(line) - if len(line) == 0: - if not is_comment: - for ln in block: - f.write(ln + "\n") - block = [] - is_comment = True - continue - is_comment = is_comment and line.startswith('#') - return f - - -def write_config_toml(writer, section_order, targets, sections): - for section in section_order: - if section == 'target': - for target in targets: - writer = write_uncommented(targets[target], writer) - else: - writer = write_uncommented(sections[section], writer) - -def quit_if_file_exists(file): - if os.path.isfile(file): - msg = "Existing '{}' detected. Exiting".format(file) - - # If the output object directory isn't empty, we can get these errors - host_objdir = os.environ.get("OBJDIR_ON_HOST") - if host_objdir is not None: - msg += "\nIs objdir '{}' clean?".format(host_objdir) - - err(msg) - -if __name__ == "__main__": - # If 'config.toml' already exists, exit the script at this point - quit_if_file_exists('config.toml') - - if "GITHUB_ACTIONS" in os.environ: - print("::group::Configure the build") - p("processing command line") - # Parse all known arguments into a configuration structure that reflects the - # TOML we're going to write out - p("") - section_order, sections, targets = parse_args(sys.argv[1:]) - - # Now that we've built up our `config.toml`, write it all out in the same - # order that we read it in. - p("") - p("writing `config.toml` in current directory") - with bootstrap.output('config.toml') as f: - write_config_toml(f, section_order, targets, sections) - - with bootstrap.output('Makefile') as f: - contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') - contents = open(contents).read() - contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') - contents = contents.replace("$(CFG_PYTHON)", sys.executable) - f.write(contents) - - p("") - p("run `python {}/x.py --help`".format(rust_dir)) - if "GITHUB_ACTIONS" in os.environ: - print("::endgroup::") diff --git a/standalonex/src/bootstrap/defaults/README.md b/standalonex/src/bootstrap/defaults/README.md deleted file mode 100644 index f5b96db1..00000000 --- a/standalonex/src/bootstrap/defaults/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# About bootstrap defaults - -These defaults are intended to be a good starting point for working with x.py, -with the understanding that no one set of defaults make sense for everyone. - -They are still experimental, and we'd appreciate your help improving them! -If you use a setting that's not in these defaults that you think -others would benefit from, please [file an issue] or make a PR with the changes. -Similarly, if one of these defaults doesn't match what you use personally, -please open an issue to get it changed. - -[file an issue]: https://github.com/rust-lang/rust/issues/new/choose diff --git a/standalonex/src/bootstrap/src/bin/git_flake_manager.rs b/standalonex/src/bootstrap/src/bin/git_flake_manager.rs new file mode 100644 index 00000000..7ab104dd --- /dev/null +++ b/standalonex/src/bootstrap/src/bin/git_flake_manager.rs @@ -0,0 +1,68 @@ +use std::path::PathBuf; +use std::fs; +use serde::Deserialize; + +use bootstrap::src::core::generate_steps::git_modules::create_branch::create_and_push_branch; + +#[derive(Debug, Deserialize)] +struct GitConfig { + base_branch: String, + new_flake_branch_prefix: String, + component: String, + arch: String, + phase: String, + step: String, + output_dir_prefix: String, +} + +#[derive(Debug, Deserialize)] +struct Config { + git: GitConfig, +} + +fn main() -> Result<(), Box> { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() // standalonex/src/bootstrap/src/bin + .unwrap() + .parent() // standalonex/src/bootstrap/src + .unwrap() + .parent() // standalonex/src/bootstrap + .unwrap() + .parent() // standalonex/src + .unwrap() + .parent() // standalonex + .unwrap() + .to_path_buf(); // rust-bootstrap-nix root + + let config_path = repo_root.join("config.toml"); + let config_content = fs::read_to_string(&config_path)?; + let config: Config = toml::from_str(&config_content)?; + + let base_branch_name = config.git.base_branch; + let branch_name = format!( + "{}/{}/{}/{}/{}", + config.git.new_flake_branch_prefix, + config.git.component, + config.git.arch, + config.git.phase, + config.git.step + ); + let output_dir = repo_root.join(format!( + "{}/{}/{}/{}/{}", + config.git.output_dir_prefix, + config.git.component, + config.git.arch, + config.git.phase, + config.git.step + )); + + create_and_push_branch( + &repo_root, + &branch_name, + &base_branch_name, + &output_dir, + false, // dry_run + )?; + + Ok(()) +} diff --git a/standalonex/src/bootstrap/src/bin/main.rs b/standalonex/src/bootstrap/src/bin/main.rs index 409a644b..15aef320 100644 --- a/standalonex/src/bootstrap/src/bin/main.rs +++ b/standalonex/src/bootstrap/src/bin/main.rs @@ -10,10 +10,9 @@ use std::io::{self, BufRead, BufReader, IsTerminal, Write}; use std::str::FromStr; use std::{env, process}; -use bootstrap::{ - Build, CONFIG_CHANGE_HISTORY, Config, Flags, Subcommand, find_recent_config_change_ids, - human_readable_changes, t, -}; +use bootstrap::{Build, CONFIG_CHANGE_HISTORY, Config, Flags, Subcommand, find_recent_config_change_ids, human_readable_changes, t, prelude::*}; +use bootstrap_config_utils::parse; +use bootstrap_config_utils::dry_run; use build_helper::ci::CiEnv; fn main() { @@ -24,115 +23,65 @@ fn main() { } let flags = Flags::parse(&args); - let config = Config::parse(flags); - - let mut build_lock; - let _build_lock_guard; - - if !config.bypass_bootstrap_lock { - // Display PID of process holding the lock - // PID will be stored in a lock file - let lock_path = config.out.join("lock"); - let pid = fs::read_to_string(&lock_path); - - build_lock = fd_lock::RwLock::new(t!(fs::OpenOptions::new() - .write(true) - .truncate(true) - .create(true) - .open(&lock_path))); - _build_lock_guard = match build_lock.try_write() { - Ok(mut lock) => { - t!(lock.write(process::id().to_string().as_ref())); - lock + let mut config = parse::parse(flags); + + // Resolve Nix paths dynamically if not already set + config.resolve_nix_paths().expect("Failed to resolve Nix paths"); + + let mut build_results = Vec::new(); + + for rustc_version in &config.rustc_versions { + for cargo_version in &config.cargo_versions { + let mut new_config = config.clone(); + new_config.initial_rustc = PathBuf::from(rustc_version); + new_config.initial_cargo = PathBuf::from(cargo_version); + + println!("Building with rustc: {} and cargo: {}", rustc_version, cargo_version); + + let mut build_lock; + let _build_lock_guard; + + if !new_config.bypass_bootstrap_lock { + // Display PID of process holding the lock + // PID will be stored in a lock file + let lock_path = new_config.out.join("lock"); + let pid = fs::read_to_string(&lock_path); + + build_lock = fd_lock::RwLock::new(t!(fs::OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(&lock_path))); + _build_lock_guard = match build_lock.try_write() { + Ok(mut lock) => { + t!(lock.write(process::id().to_string().as_ref())); + lock + } + err => { + drop(err); + if let Ok(pid) = pid { + println!("WARNING: build directory locked by process {pid}, waiting for lock"); + } else { + println!("WARNING: build directory locked, waiting for lock"); + } + let mut lock = t!(build_lock.write()); + t!(lock.write(process::id().to_string().as_ref())); + lock + } + }; } - err => { - drop(err); - if let Ok(pid) = pid { - println!("WARNING: build directory locked by process {pid}, waiting for lock"); - } else { - println!("WARNING: build directory locked, waiting for lock"); - } - let mut lock = t!(build_lock.write()); - t!(lock.write(process::id().to_string().as_ref())); - lock - } - }; - } - - // check_version warnings are not printed during setup, or during CI - let changelog_suggestion = if matches!(config.cmd, Subcommand::Setup { .. }) || CiEnv::is_ci() { - None - } else { - check_version(&config) - }; - - // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the - // changelog warning, not the `x.py setup` message. - let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); - if suggest_setup { - println!("WARNING: you have not made a `config.toml`"); - println!( - "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); - let dump_bootstrap_shims = config.dump_bootstrap_shims; - let out_dir = config.out.clone(); - - Build::new(config).build(); - if suggest_setup { - println!("WARNING: you have not made a `config.toml`"); - println!( - "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - // Give a warning if the pre-commit script is in pre-commit and not pre-push. - // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. - // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. - // Instead, look for this comment, which is almost certainly not in any custom hook. - if fs::read_to_string(pre_commit).map_or(false, |contents| { - contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") - }) { - println!( - "WARNING: You have the pre-push script installed to .git/hooks/pre-commit. \ - Consider moving it to .git/hooks/pre-push instead, which runs less often." - ); - } + let build_result = std::panic::catch_unwind(|| { + Build::new(new_config).build(); + }); - if suggest_setup || changelog_suggestion.is_some() { - println!("NOTE: this message was printed twice to make it more likely to be seen"); + build_results.push((rustc_version.clone(), cargo_version.clone(), build_result.is_ok())); + } } - if dump_bootstrap_shims { - let dump_dir = out_dir.join("bootstrap-shims-dump"); - assert!(dump_dir.exists()); - - for entry in walkdir::WalkDir::new(&dump_dir) { - let entry = t!(entry); - - if !entry.file_type().is_file() { - continue; - } - - let file = t!(fs::File::open(entry.path())); - - // To ensure deterministic results we must sort the dump lines. - // This is necessary because the order of rustc invocations different - // almost all the time. - let mut lines: Vec = t!(BufReader::new(&file).lines().collect()); - lines.sort_by_key(|t| t.to_lowercase()); - let mut file = t!(OpenOptions::new().write(true).truncate(true).open(entry.path())); - t!(file.write_all(lines.join("\n").as_bytes())); - } + println!("Build results:"); + for (rustc_version, cargo_version, success) in &build_results { + println!(" rustc: {}, cargo: {}, success: {}", rustc_version, cargo_version, success); } } @@ -176,7 +125,7 @@ fn check_version(config: &Config) -> Option { "update `config.toml` to use `change-id = {latest_change_id}` instead" )); - if io::stdout().is_terminal() && !config.dry_run() { + if io::stdout().is_terminal() && !dry_run::dry_run(&config) { t!(fs::write(warned_id_path, latest_change_id.to_string())); } } else { diff --git a/standalonex/src/bootstrap/src/bin/nix_bootstrap.rs b/standalonex/src/bootstrap/src/bin/nix_bootstrap.rs new file mode 100644 index 00000000..68326f4c --- /dev/null +++ b/standalonex/src/bootstrap/src/bin/nix_bootstrap.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello from nix_bootstrap!"); +} diff --git a/standalonex/src/bootstrap/src/bin/rustc.rs b/standalonex/src/bootstrap/src/bin/rustc.rs index 88595ff7..bd30991d 100644 --- a/standalonex/src/bootstrap/src/bin/rustc.rs +++ b/standalonex/src/bootstrap/src/bin/rustc.rs @@ -52,10 +52,15 @@ fn main() { ("RUSTC_REAL", "RUSTC_LIBDIR") }; - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + let sysroot = env::var_os("RUSTC_SYSROOT").unwrap_or_else(|| { + eprintln!("FATAL: RUSTC_SYSROOT was not set."); + eprintln!(" This environment variable is required by the rustc shim to locate the Rust standard library."); + eprintln!(" Please ensure RUSTC_SYSROOT is set to the correct path of the Rust toolchain sysroot."); + std::process::exit(1); + }); let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); - let rustc_real = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); + let rustc_real = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set!", rustc)); let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); let mut dylib_path = dylib_path(); dylib_path.insert(0, PathBuf::from(&libdir)); @@ -66,7 +71,7 @@ fn main() { // FIXME: We might want to consider removing RUSTC_REAL and setting RUSTC directly? // NOTE: we intentionally pass the name of the host, not the target. let host = env::var("CFG_COMPILER_BUILD_TRIPLE").unwrap(); - let is_clippy = args[0].to_string_lossy().ends_with(&exe("clippy-driver", &host)); + let is_clippy = args[0].to_string_lossy().ends_with(&exe("clippy-driver", &host[..])); let rustc_driver = if is_clippy { if is_build_script { // Don't run clippy on build scripts (for one thing, we may not have libstd built with @@ -82,7 +87,7 @@ fn main() { // don't remove the first arg if we're being run as RUSTC instead of RUSTC_WRAPPER. // Cargo also sometimes doesn't pass the `.exe` suffix on Windows - add it manually. let current_exe = env::current_exe().expect("couldn't get path to rustc shim"); - let arg0 = exe(args[0].to_str().expect("only utf8 paths are supported"), &host); + let arg0 = exe(args[0].to_str().expect("only utf8 paths are supported"), &host[..]); if Path::new(&arg0) == current_exe { args.remove(0); } diff --git a/standalonex/src/bootstrap/src/bin/stage1_bootstrap.rs b/standalonex/src/bootstrap/src/bin/stage1_bootstrap.rs new file mode 100644 index 00000000..221f27f8 --- /dev/null +++ b/standalonex/src/bootstrap/src/bin/stage1_bootstrap.rs @@ -0,0 +1,41 @@ +use serde::{Serialize, Deserialize}; +use std::process::Command; + +#[derive(Serialize, Deserialize, Debug)] +struct NixPaths { + rustc_path: String, + cargo_path: String, + // Add other paths as needed +} + +fn get_command_path(command_name: &str) -> Result { + let output = Command::new("which") + .arg(command_name) + .output() + .map_err(|e| format!("Failed to execute 'which {}': {}", command_name, e))?; + + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } else { + Err(format!("Command '{}' not found in PATH", command_name)) + } +} + +fn main() -> Result<(), String> { + println!("Stage 1 Booster Bootstrap: Assessing configuration..."); + + let rustc_path = get_command_path("rustc")?; + let cargo_path = get_command_path("cargo")?; + + let nix_paths = NixPaths { + rustc_path, + cargo_path, + }; + + let json_output = serde_json::to_string_pretty(&nix_paths) + .map_err(|e| format!("Failed to serialize NixPaths to JSON: {}", e))?; + + println!("{}", json_output); + + Ok(()) +} diff --git a/standalonex/src/bootstrap/src/build_impl_config.rs b/standalonex/src/bootstrap/src/build_impl_config.rs new file mode 100644 index 00000000..7192295a --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_config.rs @@ -0,0 +1,270 @@ +use std::collections::{BTreeSet, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::process::Command; + +use crate::Build; +use crate::Compiler; +use crate::GitRepo; +use crate::core::config::lld_mode::LldMode; +use crate::core::config::llvm_lib_unwind::LlvmLibunwind; +use crate::core::config::target_selection::TargetSelection; +use crate::utils::channel::GitInfo; +use crate::utils::helpers::{output, libdir}; + +impl Build { + /// Gets the space-separated set of activated features for the standard library. + /// This can be configured with the `std-features` key in config.toml. + pub fn std_features(&self, target: TargetSelection) -> String { + let mut features: BTreeSet<&str> = + self.config.rust_std_features.iter().map(|s| s.as_str()).collect(); + + match self.config.llvm_libunwind(target) { + LlvmLibunwind::InTree => features.insert("llvm-libunwind"), + LlvmLibunwind::System => features.insert("system-llvm-libunwind"), + LlvmLibunwind::No => false, + }; + + if self.config.backtrace { + features.insert("backtrace"); + } + if self.config.profiler_enabled(target) { + features.insert("profiler"); + } + // Generate memcpy, etc. FIXME: Remove this once compiler-builtins + // automatically detects this target. + if target.contains("zkvm") { + features.insert("compiler-builtins-mem"); + } + + features.into_iter().collect::>().join(" ") + } + + /// Gets the space-separated set of activated features for the compiler. + pub fn rustc_features(&self, kind: crate::core::builder::Kind, target: TargetSelection, crates: &[String]) -> String { + let possible_features_by_crates: HashSet<_> = crates + .iter() + .flat_map(|krate| &self.crates[krate].features) + .map(std::ops::Deref::deref) + .collect(); + let check = |feature: &str| -> bool { + crates.is_empty() || possible_features_by_crates.contains(feature) + }; + let mut features = vec![]; + if self.config.jemalloc && check("jemalloc") { + features.push("jemalloc"); + } + if (self.config.llvm_enabled(target) || kind == crate::core::builder::Kind::Check) && check("llvm") { + features.push("llvm"); + } + // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` + if self.config.rust_randomize_layout { + features.push("rustc_randomized_layouts"); + } + + // If debug logging is on, then we want the default for tracing: + // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 + // which is everything (including debug/trace/etc.) + // if its unset, if debug_assertions is on, then debug_logging will also be on + // as well as tracing *ignoring* this feature when debug_assertions is on + if !self.config.rust_debug_logging && check("max_level_info") { + features.push("max_level_info"); + } + + features.join(" ") + } + + /// Component directory that Cargo will produce output into (e.g. + /// release/debug) + pub fn cargo_dir(&self) -> &'static str { + if self.config.rust_optimize.is_release() { "release" } else { "debug" } + } + + pub fn tools_dir(&self, compiler: Compiler) -> PathBuf { + let out = self.out.join(compiler.host).join(format!("stage{}-tools-bin", compiler.stage)); + t!(fs::create_dir_all(&out)); + out + } + + /// Returns the root directory for all output generated in a particular + /// stage when running with a particular host compiler. + /// + /// The mode indicates what the root directory is for. + pub fn stage_out(&self, compiler: Compiler, mode: crate::Mode) -> PathBuf { + let suffix = match mode { + crate::Mode::Std => "-std", + crate::Mode::Rustc => "-rustc", + crate::Mode::Codegen => "-codegen", + crate::Mode::ToolBootstrap => "-bootstrap-tools", + crate::Mode::ToolStd | crate::Mode::ToolRustc => "-tools", + }; + self.out.join(compiler.host).join(format!("stage{}{}", compiler.stage, suffix)) + } + + /// Returns the root output directory for all Cargo output in a given stage, + /// running a particular compiler, whether or not we're building the + /// standard library, and targeting the specified architecture. + pub fn cargo_out(&self, compiler: Compiler, mode: crate::Mode, target: TargetSelection) -> PathBuf { + self.stage_out(compiler, mode).join(target).join(self.cargo_dir()) + } + + /// Root output directory of LLVM for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + pub fn llvm_out(&self, target: TargetSelection) -> PathBuf { + if self.config.llvm_from_ci && self.config.build == target { + self.config.ci_llvm_root() + } else { + self.out.join(target).join("llvm") + } + } + + pub fn enzyme_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("enzyme") + } + + pub fn gcc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("gcc") + } + + pub fn lld_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("lld") + } + + /// Output directory for all documentation for a target + pub fn doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("doc") + } + + /// Output directory for all JSON-formatted documentation for a target + pub fn json_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("json-doc") + } + + pub fn test_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("test") + } + + /// Output directory for all documentation for a target + pub fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("compiler-doc") + } + + /// Output directory for some generated md crate documentation for a target (temporary) + pub fn md_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("md-doc") + } + + /// Returns `true` if this is an external version of LLVM not managed by bootstrap. + /// In particular, we expect llvm sources to be available when this is false. + /// + /// NOTE: this is not the same as `!is_rust_llvm` when `llvm_has_patches` is set. + pub fn is_system_llvm(&self, target: TargetSelection) -> bool { + match self.config.target_config.get(&target) { + Some(crate::core::config::Target { llvm_config: Some(_), .. }) => { + let ci_llvm = self.config.llvm_from_ci && target == self.config.build; + !ci_llvm + } + // We're building from the in-tree src/llvm-project sources. + Some(crate::core::config::Target { llvm_config: None, .. }) => false, + None => false, + } + } + + /// Returns `true` if this is our custom, patched, version of LLVM. + /// + /// This does not necessarily imply that we're managing the `llvm-project` submodule. + pub fn is_rust_llvm(&self, target: TargetSelection) -> bool { + match self.config.target_config.get(&target) { + // We're using a user-controlled version of LLVM. The user has explicitly told us whether the version has our patches. + // (They might be wrong, but that's not a supported use-case.) + // In particular, this tries to support `submodules = false` and `patches = false`, for using a newer version of LLVM that's not through `rust-lang/llvm-project`. + Some(crate::core::config::Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, + // The user hasn't promised the patches match. + // This only has our patches if it's downloaded from CI or built from source. + _ => !self.is_system_llvm(target), + } + } + + /// Returns the path to llvm/bin + pub fn llvm_bin(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = output(Command::new(s).arg("--bindir")); + PathBuf::from(llvm_bindir.trim()) + } else { + self.llvm_out(self.config.build).join("bin") + } + } + + /// Returns the path to `FileCheck` binary for the specified target + pub fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { + s.to_path_buf() + } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = crate::utils::exec::command(s).arg("--bindir").run_capture_stdout(self).stdout(); + let filecheck = Path::new(llvm_bindir.trim()).join(crate::utils::helpers::exe("FileCheck", target)); + if filecheck.exists() { + filecheck + } else { + // On Fedora the system LLVM installs FileCheck in the + // llvm subdirectory of the libdir. + let llvm_libdir = crate::utils::exec::command(s).arg("--libdir").run_capture_stdout(self).stdout(); + let lib_filecheck = + Path::new(llvm_libdir.trim()).join("llvm").join(crate::utils::helpers::exe("FileCheck", target)); + if lib_filecheck.exists() { + lib_filecheck + } else { + // Return the most normal file name, even though + // it doesn't exist, so that any error message + // refers to that. + filecheck + } + } + } else { + let base = self.llvm_out(target).join("build"); + let base = if !self.ninja() && target.is_msvc() { + if self.config.llvm_optimize { + if self.config.llvm_release_debuginfo { + base.join("RelWithDebInfo") + } else { + base.join("Release") + } + } else { + base.join("Debug") + } + } else { + base + }; + base.join("bin").join(crate::utils::helpers::exe("FileCheck", target)) + } + } + + /// Directory for libraries built from C/C++ code and shared between stages. + pub fn native_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("native") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + pub fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { + self.native_dir(target).join("rust-test-helpers") + } + + /// Returns the libdir of the snapshot compiler. + pub fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc_snapshot_sysroot().join(libdir(self.config.build)) + } + + /// Returns the sysroot of the snapshot compiler. + pub fn rustc_snapshot_sysroot(&self) -> &Path { + static SYSROOT_CACHE: OnceLock = OnceLock::new(); + SYSROOT_CACHE.get_or_init(|| { + let mut rustc = Command::new(&self.initial_rustc); + rustc.args(["--print", "sysroot"]); + output(&mut rustc).trim().into() + }) + } +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/build_impl_main.rs b/standalonex/src/bootstrap/src/build_impl_main.rs new file mode 100644 index 00000000..50e44218 --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_main.rs @@ -0,0 +1,180 @@ +use build_helper::ci::gha; +use build_helper::exit; +use crate::core::builder; +use crate::core::config::dry_run::DryRun; +use crate::Subcommand; +use crate::Build; + +impl Build { + /// Executes the entire build, as configured by the flags and configuration. + pub fn build(&mut self) { + unsafe { + crate::utils::job::setup(self); + } + + // Download rustfmt early so that it can be used in rust-analyzer configs. + let _ = &builder::Builder::new(self).initial_rustfmt(); + + // hardcoded subcommands + match &self.config.cmd { + Subcommand::Format { check, all } => { + return core::build_steps::format::format( + &builder::Builder::new(self), + *check, + all, + &self.config.paths, + ); + } + Subcommand::Suggest { run } => { + return core::build_steps::suggest::suggest(&builder::Builder::new(self), run); + } + Subcommand::Perf { .. } => { + return core::build_steps::perf::perf(&builder::Builder::new(self)); + } + _ => (), + } + + if !self.config.dry_run { + { + // We first do a dry-run. This is a sanity-check to ensure that + // steps don't do anything expensive in the dry-run. + self.config.dry_run = DryRun::SelfCheck; + let builder = builder::Builder::new(self); + builder.execute_cli(); + } + self.config.dry_run = DryRun::Disabled; + let builder = builder::Builder::new(self); + builder.execute_cli(); + } else { + let builder = builder::Builder::new(self); + builder.execute_cli(); + } + + // Check for postponed failures from `test --no-fail-fast`. + let failures = self.delayed_failures.borrow(); + if failures.len() > 0 { + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in failures.iter() { + eprintln!(" - {}\n", failure); + } + exit!(1); + } + + #[cfg(feature = "build-metrics")] + self.metrics.persist(self); + } + + #[track_caller] + fn group(&self, msg: &str) -> Option { + match self.config.dry_run { + DryRun::SelfCheck => None, + DryRun::Disabled | DryRun::UserSelected => Some(gha::group(msg)), + } + } + + /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: impl Into>, + target: impl Into>, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); + let msg = if let Some(target) = target.into() { + let host = host.into().unwrap(); + if host == target { + msg(format_args!(" ({target})")) + } else { + msg(format_args!(" ({host} -> {target})")) + } + } else { + msg(format_args!("")) + }; + self.group(&msg) + } + + /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_unstaged( + &self, + action: impl Into, + what: impl Display, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = format!("{action} {what} for {target}"); + self.group(&msg) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_sysroot_tool( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: TargetSelection, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} {what} {fmt}"); + let msg = if host == target { + msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) + } else { + msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) + }; + self.group(&msg) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_clippy( + &self, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(builder::Kind::Clippy, self.config.stage, what, self.config.build, target) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_check( + &self, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(builder::Kind::Check, self.config.stage, what, self.config.build, target) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_doc( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into> + Copy, + ) -> Option { + self.msg(builder::Kind::Doc, compiler.stage, what, compiler.host, target.into()) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_build( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(builder::Kind::Build, compiler.stage, what, compiler.host, target) + } +} diff --git a/standalonex/src/bootstrap/src/build_impl_new.rs b/standalonex/src/bootstrap/src/build_impl_new.rs new file mode 100644 index 00000000..d35328db --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_new.rs @@ -0,0 +1,224 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs}; +use std::cell::RefCell; +use std::collections::HashMap; + +use build_helper::ci::gha; +use build_helper::exit; +use crate::core::config::Config; +use crate::core::config::dry_run::DryRun; +use crate::core::sanity; +use crate::core::metadata; +use crate::utils::helpers::{output, symlink_dir, dir_is_empty, exe}; +use crate::utils::channel::GitInfo; +use crate::enums::DocTests; +use crate::Subcommand; +use crate::Build; + +impl Build { + /// Creates a new set of build configuration from the `flags` on the command + /// line and the filesystem `config`. + /// + /// By default all build output will be placed in the current directory. + pub fn new(mut config: Config) -> Build { + let src = config.src.clone(); + let out = config.out.clone(); + + #[cfg(unix)] + // keep this consistent with the equivalent check in x.py: + // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 + let is_sudo = match env::var_os("SUDO_USER") { + Some(_sudo_user) => { + // SAFETY: getuid() system call is always successful and no return value is reserved + // to indicate an error. + /// + /// For more context, see https://man7.org/linux/man-pages/man2/geteuid.2.html + let uid = unsafe { libc::getuid() }; + uid == 0 + } + None => false, + }; + #[cfg(not(unix))] + let is_sudo = false; + + let rust_info = config.rust_info.clone(); + let cargo_info = config.cargo_info.clone(); + let rust_analyzer_info = config.rust_analyzer_info.clone(); + let clippy_info = config.clippy_info.clone(); + let miri_info = config.miri_info.clone(); + let rustfmt_info = config.rustfmt_info.clone(); + let enzyme_info = config.enzyme_info.clone(); + let in_tree_llvm_info = config.in_tree_llvm_info.clone(); + let in_tree_gcc_info = config.in_tree_gcc_info.clone(); + + let initial_target_libdir_str = if config.dry_run { + "/dummy/lib/path/to/lib/".to_string() + } else { + output( + Command::new(&config.initial_rustc) + .arg("--target") + .arg(config.build.rustc_target_arg()) + .arg("--print") + .arg("target-libdir"), + ) + }; + let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); + let initial_lld = initial_target_dir.join("bin").join("rust-lld"); + + let initial_sysroot = if config.dry_run { + "/dummy".to_string() + } else { + output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) + } + .trim() + .to_string(); + + // FIXME(Zalathar): Determining this path occasionally fails locally for + // unknown reasons, so we print some extra context to help track down why. + let find_initial_libdir = || { + let initial_libdir = + initial_target_dir.parent()?.parent()?.strip_prefix(&initial_sysroot).ok()?; + Some(initial_libdir.to_path_buf()) + }; + let Some(initial_libdir) = find_initial_libdir() else { + panic!( + "couldn't determine `initial_libdir`:\n- config.initial_rustc: {rustc:?}\n- initial_target_libdir_str: {initial_target_libdir_str:?}\n- initial_target_dir: {initial_target_dir:?}\n- initial_sysroot: {initial_sysroot:?}\n", + rustc = config.initial_rustc, + ); + }; + + let version = std::fs::read_to_string(src.join("src").join("version")) + .expect("failed to read src/version"); + let version = version.trim(); + + let mut bootstrap_out = std::env::current_exe () + .expect("could not determine path to running process") + .parent() + .unwrap() + .to_path_buf(); + // Since bootstrap is hardlink to deps/bootstrap-*, Solaris can sometimes give + // path with deps/ which is bad and needs to be avoided. + if bootstrap_out.ends_with("deps") { + bootstrap_out.pop(); + } + // if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { + // // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + // panic!( + // "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", + // bootstrap_out.display() + // ) + // } + + if rust_info.is_from_tarball() && config.description.is_none() { + config.description = Some("built from a source tarball".to_owned()); + } + + let mut build = Build { + initial_rustc: config.initial_rustc.clone(), + initial_cargo: config.initial_cargo.clone(), + initial_lld, + initial_libdir, + initial_sysroot: initial_sysroot.into(), + local_rebuild: config.local_rebuild, + fail_fast: config.cmd.fail_fast(), + doc_tests: config.cmd.doc_tests(), + verbosity: config.verbose, + + build: config.build, + hosts: config.hosts.clone(), + targets: config.targets.clone(), + + config, + version: version.to_string(), + src, + out, + bootstrap_out, + + cargo_info, + rust_analyzer_info, + clippy_info, + miri_info, + rustfmt_info, + enzyme_info, + in_tree_llvm_info, + in_tree_gcc_info, + cc: RefCell::new(HashMap::new()), + cxx: RefCell::new(HashMap::new()), + ar: RefCell::new(HashMap::new()), + ranlib: RefCell::new(HashMap::new()), + crates: HashMap::new(), + crate_paths: HashMap::new(), + is_sudo, + delayed_failures: RefCell::new(Vec::new()), + prerelease_version: std::cell::Cell::new(None), + + #[cfg(feature = "build-metrics")] + metrics: crate::utils::metrics::BuildMetrics::init(), + }; + + // If local-rust is the same major.minor as the current version, then force a + // local-rebuild + let local_version_verbose = + output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); + let local_release = local_version_verbose + .lines() + .filter_map(|x| x.strip_prefix("release:")) + .next() + .unwrap() + .trim(); + if local_release.split('.').take(2).eq(version.split('.').take(2)) { + build.verbose(|| println!("auto-detected local-rebuild {local_release}")); + build.local_rebuild = true; + } + + build.verbose(|| println!("finding compilers")); + crate::utils::cc_detect::find(&build); + // When running `setup`, the profile is about to change, so any requirements we have now may + // be different on the next invocation. Don't check for them until the next time x.py is + // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. + // + // Similarly, for `setup` we don't actually need submodules or cargo metadata. + if !matches!(build.config.cmd, Subcommand::Setup { .. }) { + build.verbose(|| println!("running sanity check")); + crate::core::sanity::check(&mut build); + + // Make sure we update these before gathering metadata so we don't get an error about missing + // Cargo.toml files. + let rust_submodules = ["library/backtrace", "library/stdarch"]; + for s in rust_submodules { + build.require_submodule( + s, + Some( + "The submodule is required for the standard library \ + and the main Cargo workspace.", + ), + ); + } + // Now, update all existing submodules. + build.update_existing_submodules(); + + build.verbose(|| println!("learning about cargo")); + crate::core::metadata::build(&mut build); + } + + // Create symbolic link to use host sysroot from a consistent path (e.g., in the rust-analyzer config file). + let build_triple = build.out.join(build.build); + t!(fs::create_dir_all(&build_triple)); + let host = build.out.join("host"); + if host.is_symlink() { + // Left over from a previous build; overwrite it. + // This matters if `build.build` has changed between invocations. + #[cfg(windows)] + t!(fs::remove_dir(&host)); + #[cfg(not(windows))] + t!(fs::remove_file(&host)); + } + t!( + symlink_dir(&build.config, &build_triple, &host), + format!("symlink_dir({} => {})", host.display(), build_triple.display()) + ); + + build + } +} diff --git a/standalonex/src/bootstrap/src/build_impl_paths.rs b/standalonex/src/bootstrap/src/build_impl_paths.rs new file mode 100644 index 00000000..1912b8ee --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_paths.rs @@ -0,0 +1,136 @@ +use std::path::PathBuf; + +use crate::Build; +use crate::Compiler; +use crate::core::config::target_selection::TargetSelection; + +impl Build { + /// Root output directory of LLVM for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + pub fn llvm_out(&self, target: TargetSelection) -> PathBuf { + if self.config.llvm_from_ci && self.config.build == target { + self.config.ci_llvm_root() + } else { + self.out.join(target).join("llvm") + } + } + + pub fn enzyme_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("enzyme") + } + + pub fn gcc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("gcc") + } + + pub fn lld_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("lld") + } + + /// Output directory for all documentation for a target + pub fn doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("doc") + } + + /// Output directory for all JSON-formatted documentation for a target + pub fn json_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("json-doc") + } + + pub fn test_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("test") + } + + /// Output directory for all documentation for a target + pub fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("compiler-doc") + } + + /// Output directory for some generated md crate documentation for a target (temporary) + pub fn md_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("md-doc") + } + + /// Returns the path to llvm/bin + pub fn llvm_bin(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = output(Command::new(s).arg("--bindir")); + PathBuf::from(llvm_bindir.trim()) + } else { + self.llvm_out(self.config.build).join("bin") + } + } + + /// Returns the path to `FileCheck` binary for the specified target + pub fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { + s.to_path_buf() + } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = command(s).arg("--bindir").run_capture_stdout(self).stdout(); + let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); + if filecheck.exists() { + filecheck + } else { + // On Fedora the system LLVM installs FileCheck in the + // llvm subdirectory of the libdir. + let llvm_libdir = command(s).arg("--libdir").run_capture_stdout(self).stdout(); + let lib_filecheck = + Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); + if lib_filecheck.exists() { + lib_filecheck + } else { + // Return the most normal file name, even though + // it doesn't exist, so that any error message + // refers to that. + filecheck + } + } + } else { + let base = self.llvm_out(target).join("build"); + let base = if !self.ninja() && target.is_msvc() { + if self.config.llvm_optimize { + if self.config.llvm_release_debuginfo { + base.join("RelWithDebInfo") + } else { + base.join("Release") + } + } else { + base.join("Debug") + } + } else { + base + }; + base.join("bin").join(exe("FileCheck", target)) + } + } + + /// Directory for libraries built from C/C++ code and shared between stages. + pub fn native_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(target).join("native") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + pub fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { + self.native_dir(target).join("rust-test-helpers") + } + + /// Returns the libdir of the snapshot compiler. + pub fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc_snapshot_sysroot().join(libdir(self.config.build)) + } + + /// Returns the sysroot of the snapshot compiler. + pub fn rustc_snapshot_sysroot(&self) -> &Path { + static SYSROOT_CACHE: OnceLock = OnceLock::new(); + SYSROOT_CACHE.get_or_init(|| { + let mut rustc = Command::new(&self.initial_rustc); + rustc.args(["--print", "sysroot"]); + output(&mut rustc).trim().into() + }) + } +} diff --git a/standalonex/src/bootstrap/src/build_impl_submodules.rs b/standalonex/src/bootstrap/src/build_impl_submodules.rs new file mode 100644 index 00000000..5a32dbb6 --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_submodules.rs @@ -0,0 +1,90 @@ +use std::path::Path; + +use build_helper::util; +use build_helper::exit; +use crate::Build; +use crate::core::config::Config; +use crate::utils::channel::GitInfo; +use crate::utils::helpers::{self, dir_is_empty}; + +impl Build { + /// Updates a submodule, and exits with a failure if submodule management + /// is disabled and the submodule does not exist. + /// + /// The given submodule name should be its path relative to the root of + /// the main repository. + /// + /// The given `err_hint` will be shown to the user if the submodule is not + /// checked out and submodule management is disabled. + pub fn require_submodule(&self, submodule: &str, err_hint: Option<&str>) { + // When testing bootstrap itself, it is much faster to ignore + // submodules. Almost all Steps work fine without their submodules. + if cfg!(test) && !self.config.submodules() { + return; + } + self.config.update_submodule(submodule); + let absolute_path = self.config.src.join(submodule); + if dir_is_empty(&absolute_path) { + let maybe_enable = if !self.config.submodules() + && self.config.rust_info.is_managed_git_subrepository() + { + "\nConsider setting `build.submodules = true` or manually initializing the submodules." + } else { + "" + }; + let err_hint = err_hint.map_or_else(String::new, |e| format!("\n{e}")); + eprintln!( + "submodule {submodule} does not appear to be checked out, " + "but it is required for this step{maybe_enable}{err_hint}" + ); + exit!(1); + } + } + + /// Updates all submodules, and exits with an error if submodule + /// management is disabled and the submodule does not exist. + pub fn require_and_update_all_submodules(&self) { + for submodule in util::parse_gitmodules(&self.src) { + self.require_submodule(submodule, None); + } + } + + /// If any submodule has been initialized already, sync it unconditionally. + /// This avoids contributors checking in a submodule change by accident. + pub fn update_existing_submodules(&self) { + // Avoid running git when there isn't a git checkout, or the user has + // explicitly disabled submodules in `config.toml`. + if !self.config.submodules() { + return; + } + let output = helpers::git(Some(&self.src)) + .args(["config", "--file"]) + .arg(".gitmodules") + .args(["--get-regexp", "path"]) + .run_capture(self) + .stdout(); + std::thread::scope(|s| { + // Look for `submodule.$name.path = $path` + // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` + for line in output.lines() { + let submodule = line.split_once(' ').unwrap().1; + let config = self.config.clone(); + s.spawn(move || { + Self::update_existing_submodule(&config, submodule); + }); + } + }); + } + + /// Updates the given submodule only if it's initialized already; nothing happens otherwise. + pub fn update_existing_submodule(config: &Config, submodule: &str) { + // Avoid running git when there isn't a git checkout. + if !config.submodules() { + return; + } + + if GitInfo::new(false, Path::new(submodule)).is_managed_git_subrepository() { + config.update_submodule(submodule); + } + } +} diff --git a/standalonex/src/bootstrap/src/build_impl_tools.rs b/standalonex/src/bootstrap/src/build_impl_tools.rs new file mode 100644 index 00000000..03917975 --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_tools.rs @@ -0,0 +1,60 @@ +use std::path::PathBuf; +use std::process::Command; + +use crate::Build; +use crate::CLang; +use crate::GitRepo; +use crate::core::config::lld_mode::LldMode; +use crate::core::config::target_selection::TargetSelection; +use crate::core::sanity; +use crate::utils::exec::{BootstrapCommand, command}; +use crate::utils::helpers::{exe, libdir, output}; + +impl Build { + /// Returns the path to the C compiler for the target specified. + pub fn cc(&self, target: TargetSelection) -> PathBuf { + if self.config.dry_run { + return PathBuf::new(); + } + self.cc.borrow()[&target].path().into() + } + + /// Returns a list of flags to pass to the C compiler for the target + /// specified. + pub fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { + if self.config.dry_run { + return Vec::new(); + } + let base = match c { + CLang::C => self.cc.borrow()[&target].clone(), + CLang::Cxx => self.cxx.borrow()[&target].clone(), + }; + + // Filter out -O and /O (the optimization flags) that we picked up from + // cc-rs because the build scripts will determine that for themselves. + let mut base = base + .args() + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) + .collect::>(); + + // If we're compiling C++ on macOS then we add a flag indicating that + // we want libc++ (more filled out than libstdc++), ensuring that + // LLVM/etc are all properly compiled. + if matches!(c, CLang::Cxx) && target.contains("apple-darwin") { + base.push("-stdlib=libc++".into()); + } + + // Work around an apparently bad MinGW / GCC optimization, + // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html + // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 + if &*target.triple == "i686-pc-windows-gnu" { + base.push("-fno-omit-frame-pointer".into()); + } + + if let Some(map_to) = self.debuginfo_map_to(which) { + let map = format!("{}=", self.src.display(), map_to); + let cc = self.cc(target); + if cc.ends_with("clang") || cc.ends_with("gcc") { + base.push(format!( \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/build_impl_utils.rs b/standalonex/src/bootstrap/src/build_impl_utils.rs new file mode 100644 index 00000000..47d2cb2e --- /dev/null +++ b/standalonex/src/bootstrap/src/build_impl_utils.rs @@ -0,0 +1,389 @@ +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; +use std::{io, str}; + +use sha2::digest::Digest; +use termcolor::{ColorChoice, StandardStream, WriteColor}; + +use crate::Build; +use crate::DependencyType; +use crate::core::config::dry_run::DryRun; +use crate::core::config::flags; +use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode}; +use crate::utils::helpers::{mtime, set_file_times}; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +impl Build { + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + command.mark_as_executed(); + if self.config.dry_run && !command.run_always { + return CommandOutput::default(); + } + + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + + self.verbose(|| { + println!("running: {command:?} (created at {created_at}, executed at {executed_at})") + }); + + let cmd = command.as_command_mut(); + cmd.stdout(stdout.stdio()); + cmd.stderr(stderr.stdio()); + + let output = cmd.output(); + + use std::fmt::Write; + + let mut message = String::new(); + let output: CommandOutput = match output { + // Command has succeeded + Ok(output) if output.status.success() => { + CommandOutput::from_output(output, stdout, stderr) + } + // Command has started, but then it failed + Ok(output) => { + writeln!( + message, + r#"\ +Command {command:?} did not execute successfully. +Expected success, got {} +Created at: {created_at} +Executed at: {executed_at}"#, + output.status, + ) + .unwrap(); + + let output: CommandOutput = CommandOutput::from_output(output, stdout, stderr); + + // If the output mode is OutputMode::Capture, we can now print the output. + // If it is OutputMode::Print, then the output has already been printed to + // stdout/stderr, and we thus don't have anything captured to print anyway. + if stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } + output + } + // The command did not even start + Err(e) => { + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + CommandOutput::did_not_start(stdout, stderr) + } + }; + + let fail = |message: &str, output: CommandOutput| -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = + output.stdout_if_present().take_if(|s| !s.trim().is_empty()) + { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = + output.stderr_if_present().take_if(|s| !s.trim().is_empty()) + { + println!("STDERR:\n{stderr}\n"); + } + println!("Command {command:?} has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + }; + + if !output.is_success() { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if self.fail_fast { + fail(message.as_str(), output); + } + + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(message); + } + BehaviorOnFailure::Exit => { + fail(message.as_str(), output); + } + BehaviorOnFailure::Ignore => { + // If failures are allowed, either the error has been printed already + // (OutputMode::Print) or the user used a capture output mode and wants to + // handle the error output on their own. + } + } + } + output + } + + /// Clear out `dir` if `input` is newer. + /// + /// After this executes, it will also ensure that `dir` exists. + pub fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { + let stamp = dir.join(".stamp"); + let mut cleared = false; + if mtime(&stamp) < mtime(input) { + self.verbose(|| println!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + cleared = true; + } else if stamp.exists() { + return cleared; + } + t!(fs::create_dir_all(dir)); + t!(File::create(stamp)); + cleared + } + + pub fn rust_info(&self) -> &GitInfo { + &self.config.rust_info + } + + /// Copies a file from `src` to `dst`. + /// + /// If `src` is a symlink, `src` will be resolved to the actual path + /// and copied to `dst` instead of the symlink itself. + pub fn resolve_symlink_and_copy(&self, src: &Path, dst: &Path) { + self.copy_link_internal(src, dst, true); + } + + /// Links a file from `src` to `dst`. + /// Attempts to use hard links if possible, falling back to copying. + /// You can neither rely on this being a copy nor it being a link, + /// so do not write to dst. + pub fn copy_link(&self, src: &Path, dst: &Path) { + self.copy_link_internal(src, dst, false); + } + + fn copy_link_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { + if self.config.dry_run { + return; + } + self.verbose_than(1, || println!("Copy/Link {src:?} to {dst:?}")); + if src == dst { + return; + } + if let Err(e) = fs::remove_file(dst) { + if cfg!(windows) && e.kind() != io::ErrorKind::NotFound { + // workaround for https://github.com/rust-lang/rust/issues/127126 + // if removing the file fails, attempt to rename it instead. + let now = t!(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)); + let _ = fs::rename(dst, format!("{}-{}", dst.display(), now.as_nanos())); + } + } + let metadata = t!(src.symlink_metadata(), format!("src = {}", src.display())); + let mut src = src.to_path_buf(); + if metadata.file_type().is_symlink() { + if dereference_symlinks { + src = t!(fs::canonicalize(src)); + } else { + let link = t!(fs::read_link(src)); + t!(self.symlink_file(link, dst)); + return; + } + } + if let Ok(()) = fs::hard_link(&src, dst) { + // Attempt to "easy copy" by creating a hard link (symlinks are privileged on windows), + // but if that fails just fall back to a slow `copy` operation. + } else { + if let Err(e) = fs::copy(&src, dst) { + panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) + } + t!(fs::set_permissions(dst, metadata.permissions())); + + // Restore file times because changing permissions on e.g. Linux using `chmod` can cause + // file access time to change. + let file_times = fs::FileTimes::new() + .set_accessed(t!(metadata.accessed())) + .set_modified(t!(metadata.modified())); + t!(set_file_times(dst, file_times)); + } + } + + /// Links the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. + /// Will attempt to use hard links if possible and fall back to copying. + pub fn cp_link_r(&self, src: &Path, dst: &Path) { + if self.config.dry_run { + return; + } + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + if t!(f.file_type()).is_dir() { + t!(fs::create_dir_all(&dst)); + self.cp_link_r(&path, &dst); + } else { + self.copy_link(&path, &dst); + } + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. + /// Will attempt to use hard links if possible and fall back to copying. + /// Unwanted files or directories can be skipped + /// by returning `false` from the filter function. + pub fn cp_link_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { + // Immediately recurse with an empty relative path + self.cp_link_filtered_recurse(src, dst, Path::new(""), filter) + } + + // Inner function does the actual work + fn cp_link_filtered_recurse( + &self, + src: &Path, + dst: &Path, + relative: &Path, + filter: &dyn Fn(&Path) -> bool, + ) { + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + let relative = relative.join(name); + // Only copy file or directory if the filter function returns true + if filter(&relative) { + if t!(f.file_type()).is_dir() { + let _ = fs::remove_dir_all(&dst); + self.create_dir(&dst); + self.cp_link_filtered_recurse(&path, &dst, &relative, filter); + } else { + let _ = fs::remove_file(&dst); + self.copy_link(&path, &dst); + } + } + } + } + + pub fn copy_link_to_folder(&self, src: &Path, dest_folder: &Path) { + let file_name = src.file_name().unwrap(); + let dest = dest_folder.join(file_name); + self.copy_link(src, &dest); + } + + pub fn install(&self, src: &Path, dstdir: &Path, perms: u32) { + if self.config.dry_run { + return; + } + let dst = dstdir.join(src.file_name().unwrap()); + self.verbose_than(1, || println!("Install {src:?} to {dst:?}")); + t!(fs::create_dir_all(dstdir)); + if !src.exists() { + panic!("ERROR: File \"{}\" not found!", src.display()); + } + self.copy_link_internal(src, &dst, true); + chmod(&dst, perms); + } + + pub fn read(&self, path: &Path) -> String { + if self.config.dry_run { + return String::new(); + } + t!(fs::read_to_string(path)) + } + + pub fn create_dir(&self, dir: &Path) { + if self.config.dry_run { + return; + } + t!(fs::create_dir_all(dir)) + } + + pub fn remove_dir(&self, dir: &Path) { + if self.config.dry_run { + return; + } + t!(fs::remove_dir_all(dir)) + } + + pub fn read_dir(&self, dir: &Path) -> impl Iterator { + let iter = match fs::read_dir(dir) { + Ok(v) => v, + Err(_) if self.config.dry_run => return vec![].into_iter(), + Err(err) => panic!("could not read dir {dir:?}: {err:?}"), + }; + iter.map(|e| t!(e)).collect::>().into_iter() + } + + pub fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { + if self.config.dry_run { return Ok(()); } + if cfg!(unix) { + std::os::unix::fs::symlink(src.as_ref(), link.as_ref()) + } /* else if cfg!(windows) { + std::os::windows::fs::symlink_file(src.as_ref(), link.as_ref()) + } */ else { + Err(io::Error::new(io::ErrorKind::Other, "symlinks not supported on this platform")) + } + } + + /// Check if verbosity is greater than the `level` + pub fn is_verbose_than(&self, level: usize) -> bool { + self.verbosity > level + } + + /// Runs a function if verbosity is greater than `level`. + pub fn verbose_than(&self, level: usize, f: impl Fn()) { + if self.is_verbose_than(level) { + f() + } + } + + pub fn info(&self, msg: &str) { + match self.config.dry_run { + DryRun::SelfCheck => (), + DryRun::Disabled | DryRun::UserSelected => { + println!("{msg}"); + } + } + } + + pub fn colored_stdout R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) + } + + pub fn colored_stderr R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) + } + + fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R + where + C: Fn(ColorChoice) -> StandardStream, + F: FnOnce(&mut dyn WriteColor) -> R, + { + let choice = match self.config.color { + flags::Color::Always => ColorChoice::Always, + flags::Color::Never => ColorChoice::Never, + flags::Color::Auto if !is_tty => ColorChoice::Never, + flags::Color::Auto => ColorChoice::Auto, + }; + let mut stream = constructor(choice); + let result = f(&mut stream); + stream.reset().unwrap(); + result + } +} diff --git a/standalonex/src/bootstrap/src/build_struct.rs b/standalonex/src/bootstrap/src/build_struct.rs new file mode 100644 index 00000000..8d1f2672 --- /dev/null +++ b/standalonex/src/bootstrap/src/build_struct.rs @@ -0,0 +1,75 @@ +use std::cell::{Cell, RefCell}; +use std::collections::{HashMap, HashSet}; +use std::path::{PathBuf}; + +use build_helper::ci::gha; +use crate::core::config::Config; +use crate::core::config::target_selection::TargetSelection; +use crate::enums::{DocTests, GitRepo}; +use crate::crate_struct::Crate; + +/// Global configuration for the build system. +/// +/// This structure transitively contains all configuration for the build system. +/// All filesystem-encoded configuration is in `config`, all flags are in +/// `flags`, and then parsed or probed information is listed in the keys below. +/// +/// This structure is a parameter of almost all methods in the build system, +/// although most functions are implemented as free functions rather than +/// methods specifically on this structure itself (to make it easier to +/// organize). +#[derive(Clone)] +pub struct Build { + /// User-specified configuration from `config.toml`. + pub config: Config, + + // Version information + pub version: String, + + // Properties derived from the above configuration + pub src: PathBuf, + pub out: PathBuf, + pub bootstrap_out: PathBuf, + pub cargo_info: GitInfo, + pub rust_analyzer_info: GitInfo, + pub clippy_info: GitInfo, + pub miri_info: GitInfo, + pub rustfmt_info: GitInfo, + pub enzyme_info: GitInfo, + pub in_tree_llvm_info: GitInfo, + pub in_tree_gcc_info: GitInfo, + pub local_rebuild: bool, + pub fail_fast: bool, + pub doc_tests: DocTests, + pub verbosity: usize, + + /// Build triple for the pre-compiled snapshot compiler. + pub build: TargetSelection, + /// Which triples to produce a compiler toolchain for. + pub hosts: Vec, + /// Which triples to build libraries (core/alloc/std/test/proc_macro) for. + pub targets: Vec, + + pub initial_rustc: PathBuf, + pub initial_cargo: PathBuf, + pub initial_lld: PathBuf, + pub initial_libdir: PathBuf, + pub initial_sysroot: PathBuf, + + // Runtime state filled in later on + // C/C++ compilers and archiver for all targets + pub cc: RefCell>, + pub cxx: RefCell>, + pub ar: RefCell>, + pub ranlib: RefCell>, + // Miscellaneous + // allow bidirectional lookups: both name -> path and path -> name + pub crates: HashMap, + pub crate_paths: HashMap, + pub is_sudo: bool, + pub delayed_failures: RefCell>, + pub prerelease_version: Cell>, + + #[cfg(feature = "build-metrics")] + pub metrics: crate::utils::metrics::BuildMetrics, +} diff --git a/standalonex/src/bootstrap/src/compiler.rs b/standalonex/src/bootstrap/src/compiler.rs new file mode 100644 index 00000000..80f0976f --- /dev/null +++ b/standalonex/src/bootstrap/src/compiler.rs @@ -0,0 +1,26 @@ +use crate::prelude::*; +use crate::Build; +use crate::TargetSelection; + +/// A structure representing a Rust compiler. +/// +/// Each compiler has a `stage` that it is associated with and a `host` that +/// corresponds to the platform the compiler runs on. This structure is used as +/// a parameter to many methods below. +#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] +pub struct Compiler { + pub stage: u32, + pub host: TargetSelection, +} + +impl Compiler { + pub fn with_stage(mut self, stage: u32) -> Compiler { + self.stage = stage; + self + } + + /// Returns `true` if this is a snapshot compiler for `build`'s configuration + pub fn is_snapshot(&self, build: &Build) -> bool { + self.stage == 0 && self.host == build.build + } +} diff --git a/standalonex/src/bootstrap/src/constants.rs b/standalonex/src/bootstrap/src/constants.rs new file mode 100644 index 00000000..f69a6b64 --- /dev/null +++ b/standalonex/src/bootstrap/src/constants.rs @@ -0,0 +1,36 @@ +use crate::Mode; + +pub const LLVM_TOOLS: &[&str] = &[ + "llvm-cov", // used to generate coverage report + "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility + "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume + "llvm-objdump", // used to disassemble programs + "llvm-profdata", // used to inspect and merge files generated by profiles + "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide + "llvm-size", // used to prints the size of the linker sections of a program + "llvm-strip", // used to discard symbols from binary files to reduce their size + "llvm-ar", // used for creating and modifying archive files + "llvm-as", // used to convert LLVM assembly to LLVM bitcode + "llvm-dis", // used to disassemble LLVM bitcode + "llvm-link", // Used to link LLVM bitcode + "llc", // used to compile LLVM bytecode + "opt", // used to optimize LLVM bytecode +]; + +/// LLD file names for all flavors. +pub const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; + +/// Extra `--check-cfg` to add when building the compiler or tools +/// (Mode restriction, config name, config values (if any)) +#[allow(clippy::type_complexity)] // It's fine for hard-coded list and type is explained above. +pub const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ + (None, "bootstrap", None), + (Some(Mode::Rustc), "llvm_enzyme", None), + (Some(Mode::Codegen), "llvm_enzyme", None), + (Some(Mode::ToolRustc), "llvm_enzyme", None), + (Some(Mode::ToolRustc), "rust_analyzer", None), + (Some(Mode::ToolStd), "rust_analyzer", None), + // Any library specific cfgs like `target_os`, `target_arch` should be put in + // priority the `[lints.rust.unexpected_cfgs.check-cfg]` table + // in the appropriate `library/{std,alloc,core}/Cargo.toml` +]; diff --git a/standalonex/src/bootstrap/src/core/build_steps/check.rs b/standalonex/src/bootstrap/src/core/build_steps/check.rs index d46c0ab7..c9bfe49a 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/check.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/check.rs @@ -10,33 +10,13 @@ use crate::core::builder::{ self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, crate_description, }; use crate::core::config::TargetSelection; -use crate::{Compiler, Mode, Subcommand}; +use crate::{Compiler, Mode, Subcommand, Kind}; +use crate::core::types::{CheckConfig, Rustc, RustcConfig}; +use crate::core::build_steps::rustc_step_common::rustc_should_run; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc - crates: Vec, - /// Override `Builder::kind` on cargo invocations. - /// - /// By default, `Builder::kind` is propagated as the subcommand to the cargo invocations. - /// However, there are cases when this is not desirable. For example, when running `x clippy $tool_name`, - /// passing `Builder::kind` to cargo invocations would run clippy on the entire compiler and library, - /// which is not useful if we only want to lint a few crates with specific rules. - override_build_kind: Option, -} -impl Std { - pub fn new_with_build_kind(target: TargetSelection, kind: Option) -> Self { - Self { target, crates: vec![], override_build_kind: kind } - } -} -impl Step for Std { +impl Step for Std { type Output = (); const DEFAULT: bool = true; @@ -46,7 +26,8 @@ impl Step for Std { fn make_run(run: RunConfig<'_>) { let crates = std_crates_for_run_make(&run); - run.builder.ensure(Std { target: run.target, crates, override_build_kind: None }); + let config = ::default_config(run.builder); + run.builder.ensure(Std { target: run.target, crates, config }); } fn run(self, builder: &Builder<'_>) { @@ -61,7 +42,7 @@ impl Step for Std { Mode::Std, SourceType::InTree, target, - self.override_build_kind.unwrap_or(builder.kind), + self.config.override_build_kind.unwrap_or(builder.kind), ); std_cargo(builder, target, compiler.stage, &mut cargo); @@ -115,7 +96,7 @@ impl Step for Std { Mode::Std, SourceType::InTree, target, - self.override_build_kind.unwrap_or(builder.kind), + self.config.override_build_kind.unwrap_or(builder.kind), ); // If we're not in stage 0, tests and examples will fail to compile @@ -147,25 +128,9 @@ impl Step for Std { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc - crates: Vec, - /// Override `Builder::kind` on cargo invocations. - /// - /// By default, `Builder::kind` is propagated as the subcommand to the cargo invocations. - /// However, there are cases when this is not desirable. For example, when running `x clippy $tool_name`, - /// passing `Builder::kind` to cargo invocations would run clippy on the entire compiler and library, - /// which is not useful if we only want to lint a few crates with specific rules. - override_build_kind: Option, -} -impl Rustc { + +impl Rustc { pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { Self::new_with_build_kind(target, builder, None) } @@ -173,29 +138,30 @@ impl Rustc { pub fn new_with_build_kind( target: TargetSelection, builder: &Builder<'_>, - kind: Option, + override_build_kind: Option, ) -> Self { let crates = builder .in_tree_crates("rustc-main", Some(target)) .into_iter() .map(|krate| krate.name.to_string()) .collect(); - Self { target, crates, override_build_kind: kind } + Rustc { target, crates, config: CheckConfig::new(override_build_kind) } } } -impl Step for Rustc { +impl Step for Rustc { type Output = (); const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") + rustc_should_run(run) } fn make_run(run: RunConfig<'_>) { let crates = run.make_run_crates(Alias::Compiler); - run.builder.ensure(Rustc { target: run.target, crates, override_build_kind: None }); + let config = ::default_config(run.builder); + run.builder.ensure(Rustc { target: run.target, crates, config }); } /// Builds the compiler. @@ -216,7 +182,7 @@ impl Step for Rustc { builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); } else { - builder.ensure(Std::new_with_build_kind(target, self.override_build_kind)); + builder.ensure(Std::new_with_build_kind(target, self.config.override_build_kind)); } let mut cargo = builder::Cargo::new( @@ -225,7 +191,7 @@ impl Step for Rustc { Mode::Rustc, SourceType::InTree, target, - self.override_build_kind.unwrap_or(builder.kind), + self.config.override_build_kind.unwrap_or(builder.kind), ); rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); @@ -293,9 +259,7 @@ impl Step for CodegenBackend { let compiler = builder.compiler(builder.top_stage, builder.config.build); let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(target, builder)); + builder.ensure(Rustc::::new(target, builder)); let mut cargo = builder::Cargo::new( builder, @@ -354,7 +318,7 @@ impl Step for RustAnalyzer { let compiler = builder.compiler(builder.top_stage, builder.config.build); let target = self.target; - builder.ensure(Rustc::new(target, builder)); + builder.ensure(Rustc::::new(target, builder)); let mut cargo = prepare_tool_cargo( builder, @@ -429,7 +393,7 @@ macro_rules! tool_check_step { let compiler = builder.compiler(builder.top_stage, builder.config.build); let target = self.target; - builder.ensure(Rustc::new(target, builder)); + builder.ensure(Rustc::::new(target, builder)); let mut cargo = prepare_tool_cargo( builder, diff --git a/standalonex/src/bootstrap/src/core/build_steps/check_if_tidy_is_installed.rs b/standalonex/src/bootstrap/src/core/build_steps/check_if_tidy_is_installed.rs new file mode 100644 index 00000000..4a35f24e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/check_if_tidy_is_installed.rs @@ -0,0 +1,6 @@ +use crate::core::builder::Builder; +use crate::utils::exec::command; + +pub fn check_if_tidy_is_installed(builder: &Builder<'_>) -> bool { + command("tidy").allow_failure().arg("--version").run_capture_stdout(builder).is_success() +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/clean.rs b/standalonex/src/bootstrap/src/core/build_steps/clean.rs index d857de96..da8cf566 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/clean.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/clean.rs @@ -97,7 +97,7 @@ clean_crate_tree! { } fn clean(build: &Build, all: bool, stage: Option) { - if build.config.dry_run() { + if build.config.dry_run { return; } diff --git a/standalonex/src/bootstrap/src/core/build_steps/clippy.rs b/standalonex/src/bootstrap/src/core/build_steps/clippy.rs index 0884d86c..febea673 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/clippy.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/clippy.rs @@ -8,6 +8,8 @@ use crate::core::build_steps::compile::std_crates_for_run_make; use crate::core::builder; use crate::core::builder::{Alias, Kind, RunConfig, Step, crate_description}; use crate::{Mode, Subcommand, TargetSelection}; +use crate::core::types::{LintConfig, Rustc, RustcConfig}; +use crate::core::build_steps::rustc_step_common::rustc_should_run; /// Disable the most spammy clippy lints const IGNORED_RULES_FOR_STD_AND_RUSTC: &[&str] = &[ @@ -85,45 +87,9 @@ pub fn get_clippy_rules_in_order(all_args: &[String], config: &LintConfig) -> Ve result.into_iter().map(|v| v.1).collect() } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct LintConfig { - pub allow: Vec, - pub warn: Vec, - pub deny: Vec, - pub forbid: Vec, -} -impl LintConfig { - fn new(builder: &Builder<'_>) -> Self { - match builder.config.cmd.clone() { - Subcommand::Clippy { allow, deny, warn, forbid, .. } => { - Self { allow, warn, deny, forbid } - } - _ => unreachable!("LintConfig can only be called from `clippy` subcommands."), - } - } - fn merge(&self, other: &Self) -> Self { - let merged = |self_attr: &[String], other_attr: &[String]| -> Vec { - self_attr.iter().cloned().chain(other_attr.iter().cloned()).collect() - }; - // This is written this way to ensure we get a compiler error if we add a new field. - Self { - allow: merged(&self.allow, &other.allow), - warn: merged(&self.warn, &other.warn), - deny: merged(&self.deny, &other.deny), - forbid: merged(&self.forbid, &other.forbid), - } - } -} -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - config: LintConfig, - /// Whether to lint only a subset of crates. - crates: Vec, -} impl Step for Std { type Output = (); @@ -175,27 +141,21 @@ impl Step for Std { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - config: LintConfig, - /// Whether to lint only a subset of crates. - crates: Vec, -} -impl Step for Rustc { + +impl Step for Rustc { type Output = (); const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") + rustc_should_run(run) } fn make_run(run: RunConfig<'_>) { let crates = run.make_run_crates(Alias::Compiler); - let config = LintConfig::new(run.builder); - run.builder.ensure(Rustc { target: run.target, config, crates }); + let config = ::default_config(run.builder); + run.builder.ensure(Rustc { target: run.target, crates, config }); } /// Lints the compiler. diff --git a/standalonex/src/bootstrap/src/core/build_steps/common_test_fields.rs b/standalonex/src/bootstrap/src/core/build_steps/common_test_fields.rs new file mode 100644 index 00000000..930546bf --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/common_test_fields.rs @@ -0,0 +1,10 @@ +use crate::core::builder::Compiler; +use crate::core::config::TargetSelection; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CommonTestFields { + pub compiler: Compiler, + pub target: TargetSelection, + pub host: TargetSelection, + pub stage: u32, +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile.rs b/standalonex/src/bootstrap/src/core/build_steps/compile.rs index e99a0265..a29da9d3 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/compile.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/compile.rs @@ -1,10 +1,7 @@ -//! Implementation of compiling various phases of the compiler and standard -//! library. -//! -//! This module contains some of the real meat in the bootstrap build system -//! which is where Cargo is used to compile the standard library, libtest, and -//! the compiler. This module is also responsible for assembling the sysroot as it -//! goes along from the output of the previous stage. +//use super::compile_modules::*; + +//! This file was automatically generated by a refactoring script. +//! It now imports modules containing definitions extracted from the original file. use std::borrow::Cow; use std::collections::HashSet; @@ -30,2300 +27,7 @@ use crate::utils::helpers::{ }; use crate::{CLang, Compiler, DependencyType, GitRepo, LLVM_TOOLS, Mode}; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Std { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build only a subset of crates in the standard library. - /// - /// This shouldn't be used from other steps; see the comment on [`Rustc`]. - crates: Vec, - /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, - /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overridden by `builder.ensure` from other steps. - force_recompile: bool, - extra_rust_args: &'static [&'static str], - is_for_mir_opt_tests: bool, -} - -impl Std { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - } - } - - pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: true, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - } - } - - pub fn new_for_mir_opt_tests(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args: &[], - is_for_mir_opt_tests: true, - } - } - - pub fn new_with_extra_rust_args( - compiler: Compiler, - target: TargetSelection, - extra_rust_args: &'static [&'static str], - ) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args, - is_for_mir_opt_tests: false, - } - } - - fn copy_extra_objects( - &self, - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, - ) -> Vec<(PathBuf, DependencyType)> { - let mut deps = Vec::new(); - if !self.is_for_mir_opt_tests { - deps.extend(copy_third_party_objects(builder, compiler, target)); - deps.extend(copy_self_contained_objects(builder, compiler, target)); - } - deps - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = std_crates_for_run_make(&run); - let builder = run.builder; - - // Force compilation of the standard library from source if the `library` is modified. This allows - // library team to compile the standard library without needing to compile the compiler with - // the `rust.download-rustc=true` option. - let force_recompile = builder.rust_info().is_managed_git_subrepository() - && builder.download_rustc() - && builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none(); - - run.builder.ensure(Std { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - force_recompile, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - }); - } - - /// Builds the standard library. - /// - /// This will build the standard library for a particular stage of the build - /// using the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - - // When using `download-rustc`, we already have artifacts for the host available. Don't - // recompile them. - if builder.download_rustc() && target == builder.build.build - // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so - // its artifacts can't be reused. - && compiler.stage != 0 - && !self.force_recompile - { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - cp_rustc_component_to_ci_sysroot( - builder, - &sysroot, - builder.config.ci_rust_std_contents(), - ); - return; - } - - if builder.config.keep_stage.contains(&compiler.stage) - || builder.config.keep_stage_std.contains(&compiler.stage) - { - builder.info("WARNING: Using a potentially old libstd. This may not behave well."); - - builder.ensure(StartupObjects { compiler, target }); - - self.copy_extra_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler)); - return; - } - - builder.require_submodule("library/stdarch", None); - - let mut target_deps = builder.ensure(StartupObjects { compiler, target }); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Std::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting library (stage{} -> stage{})", - compiler_to_use.stage, compiler.stage - ) - } else { - format!( - "Uplifting library (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, compiler_to_use.host, compiler.stage, target - ) - }; - builder.info(&msg); - - // Even if we're not building std this stage, the new sysroot must - // still contain the third party objects needed by various targets. - self.copy_extra_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler_to_use)); - return; - } - - target_deps.extend(self.copy_extra_objects(builder, &compiler, target)); - - // The LLD wrappers and `rust-lld` are self-contained linking components that can be - // necessary to link the stdlib on some targets. We'll also need to copy these binaries to - // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. - if compiler.stage == 0 && compiler.host == builder.config.build { - // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. - let src_sysroot_bin = builder - .rustc_snapshot_sysroot() - .join("lib") - .join("rustlib") - .join(compiler.host) - .join("bin"); - if src_sysroot_bin.exists() { - let target_sysroot_bin = builder.sysroot_target_bindir(compiler, target); - t!(fs::create_dir_all(&target_sysroot_bin)); - builder.cp_link_r(&src_sysroot_bin, &target_sysroot_bin); - } - } - - // We build a sysroot for mir-opt tests using the same trick that Miri does: A check build - // with -Zalways-encode-mir. This frees us from the need to have a target linker, and the - // fact that this is a check build integrates nicely with run_cargo. - let mut cargo = if self.is_for_mir_opt_tests { - let mut cargo = builder::Cargo::new_for_mir_opt_tests( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - Kind::Check, - ); - cargo.rustflag("-Zalways-encode-mir"); - cargo.arg("--manifest-path").arg(builder.src.join("library/sysroot/Cargo.toml")); - cargo - } else { - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - Kind::Build, - ); - std_cargo(builder, target, compiler.stage, &mut cargo); - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - cargo - }; - - // See src/bootstrap/synthetic_targets.rs - if target.is_synthetic() { - cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); - } - for rustflag in self.extra_rust_args.iter() { - cargo.rustflag(rustflag); - } - - let _guard = builder.msg( - Kind::Build, - compiler.stage, - format_args!("library artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - run_cargo( - builder, - cargo, - vec![], - &libstd_stamp(builder, compiler, target), - target_deps, - self.is_for_mir_opt_tests, // is_check - false, - ); - - builder.ensure(StdLink::from_std( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -fn copy_and_stamp( - builder: &Builder<'_>, - libdir: &Path, - sourcedir: &Path, - name: &str, - target_deps: &mut Vec<(PathBuf, DependencyType)>, - dependency_type: DependencyType, -) { - let target = libdir.join(name); - builder.copy_link(&sourcedir.join(name), &target); - - target_deps.push((target, dependency_type)); -} - -fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { - let libunwind_path = builder.ensure(llvm::Libunwind { target }); - let libunwind_source = libunwind_path.join("libunwind.a"); - let libunwind_target = libdir.join("libunwind.a"); - builder.copy_link(&libunwind_source, &libunwind_target); - libunwind_target -} - -/// Copies third party objects needed by various targets. -fn copy_third_party_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let mut target_deps = vec![]; - - if builder.config.needs_sanitizer_runtime_built(target) && compiler.stage != 0 { - // The sanitizers are only copied in stage1 or above, - // to avoid creating dependency on LLVM. - target_deps.extend( - copy_sanitizers(builder, compiler, target) - .into_iter() - .map(|d| (d, DependencyType::Target)), - ); - } - - if target == "x86_64-fortanix-unknown-sgx" - || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree - && (target.contains("linux") || target.contains("fuchsia")) - { - let libunwind_path = - copy_llvm_libunwind(builder, target, &builder.sysroot_target_libdir(*compiler, target)); - target_deps.push((libunwind_path, DependencyType::Target)); - } - - target_deps -} - -/// Copies third party objects needed by various targets for self-contained linkage. -fn copy_self_contained_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let libdir_self_contained = - builder.sysroot_target_libdir(*compiler, target).join("self-contained"); - t!(fs::create_dir_all(&libdir_self_contained)); - let mut target_deps = vec![]; - - // Copies the libc and CRT objects. - // - // rustc historically provides a more self-contained installation for musl targets - // not requiring the presence of a native musl toolchain. For example, it can fall back - // to using gcc from a glibc-targeting toolchain for linking. - // To do that we have to distribute musl startup objects as a part of Rust toolchain - // and link with them manually in the self-contained mode. - if target.contains("musl") && !target.contains("unikraft") { - let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { - panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) - }); - for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); - for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { - let src = crt_path.join(obj); - let target = libdir_self_contained.join(obj); - builder.copy_link(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - - if !target.starts_with("s390x") { - let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); - target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); - } - } else if target.contains("-wasi") { - let srcdir = builder.wasi_libdir(target).unwrap_or_else(|| { - panic!( - "Target {:?} does not have a \"wasi-root\" key in Config.toml \ - or `$WASI_SDK_PATH` set", - target.triple - ) - }); - for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - } else if target.is_windows_gnu() { - for obj in ["crt2.o", "dllcrt2.o"].iter() { - let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); - let target = libdir_self_contained.join(obj); - builder.copy_link(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - } - - target_deps -} - -/// Resolves standard library crates for `Std::run_make` for any build kind (like check, build, clippy, etc.). -pub fn std_crates_for_run_make(run: &RunConfig<'_>) -> Vec { - // FIXME: Extend builder tests to cover the `crates` field of `Std` instances. - if cfg!(feature = "bootstrap-self-test") { - return vec![]; - } - - let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); - let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false); - - // For no_std targets, do not add any additional crates to the compilation other than what `compile::std_cargo` already adds for no_std targets. - if target_is_no_std { - vec![] - } - // If the paths include "library", build the entire standard library. - else if has_alias { - run.make_run_crates(builder::Alias::Library) - } else { - run.cargo_crates_in_set() - } -} - -/// Tries to find LLVM's `compiler-rt` source directory, for building `library/profiler_builtins`. -/// -/// Normally it lives in the `src/llvm-project` submodule, but if we will be using a -/// downloaded copy of CI LLVM, then we try to use the `compiler-rt` sources from -/// there instead, which lets us avoid checking out the LLVM submodule. -fn compiler_rt_for_profiler(builder: &Builder<'_>) -> PathBuf { - // Try to use `compiler-rt` sources from downloaded CI LLVM, if possible. - if builder.config.llvm_from_ci { - // CI LLVM might not have been downloaded yet, so try to download it now. - builder.config.maybe_download_ci_llvm(); - let ci_llvm_compiler_rt = builder.config.ci_llvm_root().join("compiler-rt"); - if ci_llvm_compiler_rt.exists() { - return ci_llvm_compiler_rt; - } - } - - // Otherwise, fall back to requiring the LLVM submodule. - builder.require_submodule("src/llvm-project", { - Some("The `build.profiler` config option requires `compiler-rt` sources from LLVM.") - }); - builder.src.join("src/llvm-project/compiler-rt") -} - -/// Configure cargo to compile the standard library, adding appropriate env vars -/// and such. -pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { - if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { - cargo.env("MACOSX_DEPLOYMENT_TARGET", target); - } - - // Paths needed by `library/profiler_builtins/build.rs`. - if let Some(path) = builder.config.profiler_path(target) { - cargo.env("LLVM_PROFILER_RT_LIB", path); - } else if builder.config.profiler_enabled(target) { - let compiler_rt = compiler_rt_for_profiler(builder); - // Currently this is separate from the env var used by `compiler_builtins` - // (below) so that adding support for CI LLVM here doesn't risk breaking - // the compiler builtins. But they could be unified if desired. - cargo.env("RUST_COMPILER_RT_FOR_PROFILER", compiler_rt); - } - - // Determine if we're going to compile in optimized C intrinsics to - // the `compiler-builtins` crate. These intrinsics live in LLVM's - // `compiler-rt` repository. - // - // Note that this shouldn't affect the correctness of `compiler-builtins`, - // but only its speed. Some intrinsics in C haven't been translated to Rust - // yet but that's pretty rare. Other intrinsics have optimized - // implementations in C which have only had slower versions ported to Rust, - // so we favor the C version where we can, but it's not critical. - // - // If `compiler-rt` is available ensure that the `c` feature of the - // `compiler-builtins` crate is enabled and it's configured to learn where - // `compiler-rt` is located. - let compiler_builtins_c_feature = if builder.config.optimized_compiler_builtins { - // NOTE: this interacts strangely with `llvm-has-rust-patches`. In that case, we enforce `submodules = false`, so this is a no-op. - // But, the user could still decide to manually use an in-tree submodule. - // - // NOTE: if we're using system llvm, we'll end up building a version of `compiler-rt` that doesn't match the LLVM we're linking to. - // That's probably ok? At least, the difference wasn't enforced before. There's a comment in - // the compiler_builtins build script that makes me nervous, though: - // https://github.com/rust-lang/compiler-builtins/blob/31ee4544dbe47903ce771270d6e3bea8654e9e50/build.rs#L575-L579 - builder.require_submodule( - "src/llvm-project", - Some( - "The `build.optimized-compiler-builtins` config option \ - requires `compiler-rt` sources from LLVM.", - ), - ); - let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); - assert!(compiler_builtins_root.exists()); - // The path to `compiler-rt` is also used by `profiler_builtins` (above), - // so if you're changing something here please also change that as appropriate. - cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); - " compiler-builtins-c" - } else { - "" - }; - - // `libtest` uses this to know whether or not to support - // `-Zunstable-options`. - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - let mut features = String::new(); - - if builder.no_std(target) == Some(true) { - features += " compiler-builtins-mem"; - if !target.starts_with("sbf") && !target.starts_with("bpf") { - features.push_str(compiler_builtins_c_feature); - } - - // for no-std targets we only compile a few no_std crates - cargo - .args(["-p", "alloc"]) - .arg("--manifest-path") - .arg(builder.src.join("library/alloc/Cargo.toml")) - .arg("--features") - .arg(features); - } else { - features += &builder.std_features(target); - features.push_str(compiler_builtins_c_feature); - - cargo - .arg("--features") - .arg(features) - .arg("--manifest-path") - .arg(builder.src.join("library/sysroot/Cargo.toml")); - - // Help the libc crate compile by assisting it in finding various - // sysroot native libraries. - if target.contains("musl") { - if let Some(p) = builder.musl_libdir(target) { - let root = format!("native={}", p.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - - if target.contains("-wasi") { - if let Some(dir) = builder.wasi_libdir(target) { - let root = format!("native={}", dir.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - } - - // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that - // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be - // built with bitcode so that the produced rlibs can be used for both LTO - // builds (which use bitcode) and non-LTO builds (which use object code). - // So we override the override here! - // - // But we don't bother for the stage 0 compiler because it's never used - // with LTO. - if stage >= 1 { - cargo.rustflag("-Cembed-bitcode=yes"); - } - if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // By default, rustc does not include unwind tables unless they are required - // for a particular target. They are not required by RISC-V targets, but - // compiling the standard library with them means that users can get - // backtraces without having to recompile the standard library themselves. - // - // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 - if target.contains("riscv") { - cargo.rustflag("-Cforce-unwind-tables=yes"); - } - - // Enable frame pointers by default for the library. Note that they are still controlled by a - // separate setting for the compiler. - cargo.rustflag("-Cforce-frame-pointers=yes"); - - let html_root = - format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); - cargo.rustflag(&html_root); - cargo.rustdocflag(&html_root); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct StdLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Vec, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl StdLink { - fn from_std(std: Std, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: std.compiler, - target: std.target, - crates: std.crates, - force_recompile: std.force_recompile, - } - } -} - -impl Step for StdLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Link all libstd rlibs/dylibs into the sysroot location. - /// - /// Links those artifacts generated by `compiler` to the `stage` compiler's - /// sysroot for the specified `host` and `target`. - /// - /// Note that this assumes that `compiler` has already generated the libstd - /// libraries for `target`, and this method will find them in the relevant - /// output directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - - // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. - let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { - // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { - compiler: self.compiler, - force_recompile: self.force_recompile, - }); - let libdir = sysroot.join(lib).join("rustlib").join(target).join("lib"); - let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host).join("lib"); - (libdir, hostdir) - } else { - let libdir = builder.sysroot_target_libdir(target_compiler, target); - let hostdir = builder.sysroot_target_libdir(target_compiler, compiler.host); - (libdir, hostdir) - }; - - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - - // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` - // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, - // and is not set to a custom path. - if compiler.stage == 0 - && builder - .build - .config - .initial_rustc - .starts_with(builder.out.join(compiler.host).join("stage0/bin")) - { - // Copy bin files from stage0/bin to stage0-sysroot/bin - let sysroot = builder.out.join(compiler.host).join("stage0-sysroot"); - - let host = compiler.host; - let stage0_bin_dir = builder.out.join(host).join("stage0/bin"); - let sysroot_bin_dir = sysroot.join("bin"); - t!(fs::create_dir_all(&sysroot_bin_dir)); - builder.cp_link_r(&stage0_bin_dir, &sysroot_bin_dir); - - // Copy all files from stage0/lib to stage0-sysroot/lib - let stage0_lib_dir = builder.out.join(host).join("stage0/lib"); - if let Ok(files) = fs::read_dir(stage0_lib_dir) { - for file in files { - let file = t!(file); - let path = file.path(); - if path.is_file() { - builder - .copy_link(&path, &sysroot.join("lib").join(path.file_name().unwrap())); - } - } - } - - // Copy codegen-backends from stage0 - let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); - t!(fs::create_dir_all(&sysroot_codegen_backends)); - let stage0_codegen_backends = builder - .out - .join(host) - .join("stage0/lib/rustlib") - .join(host) - .join("codegen-backends"); - if stage0_codegen_backends.exists() { - builder.cp_link_r(&stage0_codegen_backends, &sysroot_codegen_backends); - } - } - } -} - -/// Copies sanitizer runtime libraries into target libdir. -fn copy_sanitizers( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec { - let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); - - if builder.config.dry_run() { - return Vec::new(); - } - - let mut target_deps = Vec::new(); - let libdir = builder.sysroot_target_libdir(*compiler, target); - - for runtime in &runtimes { - let dst = libdir.join(&runtime.name); - builder.copy_link(&runtime.path, &dst); - - // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for - // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do - // not list them here to rename and sign the runtime library. - if target == "x86_64-apple-darwin" - || target == "aarch64-apple-darwin" - || target == "aarch64-apple-ios" - || target == "aarch64-apple-ios-sim" - || target == "x86_64-apple-ios" - { - // Update the library’s install name to reflect that it has been renamed. - apple_darwin_update_library_name(builder, &dst, &format!("@rpath/{}", runtime.name)); - // Upon renaming the install name, the code signature of the file will invalidate, - // so we will sign it again. - apple_darwin_sign_file(builder, &dst); - } - - target_deps.push(dst); - } - - target_deps -} - -fn apple_darwin_update_library_name(builder: &Builder<'_>, library_path: &Path, new_name: &str) { - command("install_name_tool").arg("-id").arg(new_name).arg(library_path).run(builder); -} - -fn apple_darwin_sign_file(builder: &Builder<'_>, file_path: &Path) { - command("codesign") - .arg("-f") // Force to rewrite the existing signature - .arg("-s") - .arg("-") - .arg(file_path) - .run(builder); -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct StartupObjects { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for StartupObjects { - type Output = Vec<(PathBuf, DependencyType)>; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("library/rtstartup") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(StartupObjects { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - }); - } - - /// Builds and prepare startup objects like rsbegin.o and rsend.o - /// - /// These are primarily used on Windows right now for linking executables/dlls. - /// They don't require any library support as they're just plain old object - /// files, so we just use the nightly snapshot compiler to always build them (as - /// no other compilers are guaranteed to be available). - fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { - let for_compiler = self.compiler; - let target = self.target; - if !target.is_windows_gnu() { - return vec![]; - } - - let mut target_deps = vec![]; - - let src_dir = &builder.src.join("library").join("rtstartup"); - let dst_dir = &builder.native_dir(target).join("rtstartup"); - let sysroot_dir = &builder.sysroot_target_libdir(for_compiler, target); - t!(fs::create_dir_all(dst_dir)); - - for file in &["rsbegin", "rsend"] { - let src_file = &src_dir.join(file.to_string() + ".rs"); - let dst_file = &dst_dir.join(file.to_string() + ".o"); - if !up_to_date(src_file, dst_file) { - let mut cmd = command(&builder.initial_rustc); - cmd.env("RUSTC_BOOTSTRAP", "1"); - if !builder.local_rebuild { - // a local_rebuild compiler already has stage1 features - cmd.arg("--cfg").arg("bootstrap"); - } - cmd.arg("--target") - .arg(target.rustc_target_arg()) - .arg("--emit=obj") - .arg("-o") - .arg(dst_file) - .arg(src_file) - .run(builder); - } - - let target = sysroot_dir.join((*file).to_string() + ".o"); - builder.copy_link(dst_file, &target); - target_deps.push((target, DependencyType::Target)); - } - - target_deps - } -} - -fn cp_rustc_component_to_ci_sysroot(builder: &Builder<'_>, sysroot: &Path, contents: Vec) { - let ci_rustc_dir = builder.config.ci_rustc_dir(); - - for file in contents { - let src = ci_rustc_dir.join(&file); - let dst = sysroot.join(file); - if src.is_dir() { - t!(fs::create_dir_all(dst)); - } else { - builder.copy_link(&src, &dst); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// The **previous** compiler used to compile this compiler. - pub compiler: Compiler, - /// Whether to build a subset of crates, rather than the whole compiler. - /// - /// This should only be requested by the user, not used within bootstrap itself. - /// Using it within bootstrap can lead to confusing situation where lints are replayed - /// in two different steps. - crates: Vec, -} - -impl Rustc { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default() } - } -} - -impl Step for Rustc { - /// We return the stage of the "actual" compiler (not the uplifted one). - /// - /// By "actual" we refer to the uplifting logic where we may not compile the requested stage; - /// instead, we uplift it from the previous stages. Which can lead to bootstrap failures in - /// specific situations where we request stage X from other steps. However we may end up - /// uplifting it from stage Y, causing the other stage to fail when attempting to link with - /// stage X which was never actually built. - type Output = u32; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let mut crates = run.builder.in_tree_crates("rustc-main", None); - for (i, krate) in crates.iter().enumerate() { - // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. - // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. - if krate.name == "rustc-main" { - crates.swap_remove(i); - break; - } - } - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.cargo_crates_in_set(); - run.builder.ensure(Rustc { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) -> u32 { - let compiler = self.compiler; - let target = self.target; - - // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, - // so its artifacts can't be reused. - if builder.download_rustc() && compiler.stage != 0 { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - cp_rustc_component_to_ci_sysroot( - builder, - &sysroot, - builder.config.ci_rustc_dev_contents(), - ); - return compiler.stage; - } - - builder.ensure(Std::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info("WARNING: Using a potentially old librustc. This may not behave well."); - builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self, compiler)); - - return compiler.stage; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Rustc::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting rustc (stage{} -> stage{})", - compiler_to_use.stage, - compiler.stage + 1 - ) - } else { - format!( - "Uplifting rustc (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, - compiler_to_use.host, - compiler.stage + 1, - target - ) - }; - builder.info(&msg); - builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); - return compiler_to_use.stage; - } - - // Ensure that build scripts and proc macros have a std / libproc_macro to link against. - builder.ensure(Std::new( - builder.compiler(self.compiler.stage, builder.config.build), - builder.config.build, - )); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Rustc, - SourceType::InTree, - target, - Kind::Build, - ); - - rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - - // NB: all RUSTFLAGS should be added to `rustc_cargo()` so they will be - // consistently applied by check/doc/test modes too. - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - if builder.build.config.enable_bolt_settings && compiler.stage == 1 { - // Relocations are required for BOLT to work. - cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); - } - - let _guard = builder.msg_sysroot_tool( - Kind::Build, - compiler.stage, - format_args!("compiler artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - let stamp = librustc_stamp(builder, compiler, target); - run_cargo( - builder, - cargo, - vec![], - &stamp, - vec![], - false, - true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. - ); - - // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain - // unexpected debuginfo from dependencies, for example from the C++ standard library used in - // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with - // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo - // away after the fact. - if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None - && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None - { - let target_root_dir = stamp.parent().unwrap(); - let rustc_driver = target_root_dir.join("librustc_driver.so"); - strip_debug(builder, target, &rustc_driver); - } - - builder.ensure(RustcLink::from_rustc( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - - compiler.stage - } -} - -pub fn rustc_cargo( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - compiler: &Compiler, - crates: &[String], -) { - cargo - .arg("--features") - .arg(builder.rustc_features(builder.kind, target, crates)) - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc/Cargo.toml")); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); - - // If the rustc output is piped to e.g. `head -n1` we want the process to be killed, rather than - // having an error bubble up and cause a panic. - // - // FIXME(jieyouxu): this flag is load-bearing for rustc to not ICE on broken pipes, because - // rustc internally sometimes uses std `println!` -- but std `println!` by default will panic on - // broken pipes, and uncaught panics will manifest as an ICE. The compiler *should* handle this - // properly, but this flag is set in the meantime to paper over the I/O errors. - // - // See for details. - // - // Also see the discussion for properly handling I/O errors related to broken pipes, i.e. safe - // variants of `println!` in - // . - cargo.rustflag("-Zon-broken-pipe=kill"); - - if builder.config.llvm_enzyme { - cargo.rustflag("-l").rustflag("Enzyme-19"); - } - - // Building with protected visibility reduces the number of dynamic relocations needed, giving - // us a faster startup time. However GNU ld < 2.40 will error if we try to link a shared object - // with direct references to protected symbols, so for now we only use protected symbols if - // linking with LLD is enabled. - if builder.build.config.lld_mode.is_used() { - cargo.rustflag("-Zdefault-visibility=protected"); - } - - // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary - // and may just be a time sink. - if compiler.stage != 0 { - match builder.config.rust_lto { - RustcLto::Thin | RustcLto::Fat => { - // Since using LTO for optimizing dylibs is currently experimental, - // we need to pass -Zdylib-lto. - cargo.rustflag("-Zdylib-lto"); - // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when - // compiling dylibs (and their dependencies), even when LTO is enabled for the - // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. - let lto_type = match builder.config.rust_lto { - RustcLto::Thin => "thin", - RustcLto::Fat => "fat", - _ => unreachable!(), - }; - cargo.rustflag(&format!("-Clto={lto_type}")); - cargo.rustflag("-Cembed-bitcode=yes"); - } - RustcLto::ThinLocal => { /* Do nothing, this is the default */ } - RustcLto::Off => { - cargo.rustflag("-Clto=off"); - } - } - } else if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // With LLD, we can use ICF (identical code folding) to reduce the executable size - // of librustc_driver/rustc and to improve i-cache utilization. - // - // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) - // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: - // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 - // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 - if builder.config.lld_mode.is_used() && !compiler.host.is_msvc() { - cargo.rustflag("-Clink-args=-Wl,--icf=all"); - } - - if builder.config.rust_profile_use.is_some() && builder.config.rust_profile_generate.is_some() { - panic!("Cannot use and generate PGO profiles at the same time"); - } - let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-generate={path}")); - // Apparently necessary to avoid overflowing the counters during - // a Cargo build profile - cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); - true - } else { - false - } - } else if let Some(path) = &builder.config.rust_profile_use { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-use={path}")); - if builder.is_verbose() { - cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); - } - true - } else { - false - } - } else { - false - }; - if is_collecting { - // Ensure paths to Rust sources are relative, not absolute. - cargo.rustflag(&format!( - "-Cllvm-args=-static-func-strip-dirname-prefix={}", - builder.config.src.components().count() - )); - } - - rustc_cargo_env(builder, cargo, target, compiler.stage); -} - -pub fn rustc_cargo_env( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - stage: u32, -) { - // Set some configuration variables picked up by build scripts and - // the compiler alike - cargo - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("CFG_VERSION", builder.rust_version()); - // Some tools like Cargo detect their own git information in build scripts. When omit-git-hash - // is enabled in config.toml, we pass this environment variable to tell build scripts to avoid - // detecting git information on their own. - if builder.config.omit_git_hash { - cargo.env("CFG_OMIT_GIT_HASH", "1"); - } - - if let Some(backend) = builder.config.default_codegen_backend(target) { - cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); - } - - let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); - let target_config = builder.config.target_config.get(&target); - - cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); - - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - // Prefer the current target's own default_linker, else a globally - // specified one. - if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { - cargo.env("CFG_DEFAULT_LINKER", s); - } else if let Some(ref s) = builder.config.rustc_default_linker { - cargo.env("CFG_DEFAULT_LINKER", s); - } - - // Enable rustc's env var for `rust-lld` when requested. - if builder.config.lld_enabled - && (builder.config.channel == "dev" || builder.config.channel == "nightly") - { - cargo.env("CFG_USE_SELF_CONTAINED_LINKER", "1"); - } - - if builder.config.rust_verify_llvm_ir { - cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); - } - - if builder.config.llvm_enzyme { - cargo.rustflag("--cfg=llvm_enzyme"); - } - - // Note that this is disabled if LLVM itself is disabled or we're in a check - // build. If we are in a check build we still go ahead here presuming we've - // detected that LLVM is already built and good to go which helps prevent - // busting caches (e.g. like #71152). - if builder.config.llvm_enabled(target) { - let building_is_expensive = - crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target, false) - .should_build(); - // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler - let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; - let should_skip_build = building_is_expensive && can_skip_build; - if !should_skip_build { - rustc_llvm_env(builder, cargo, target) - } - } -} - -/// Pass down configuration from the LLVM build into the build of -/// rustc_llvm and rustc_codegen_llvm. -fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { - if builder.is_rust_llvm(target) { - cargo.env("LLVM_RUSTLLVM", "1"); - } - let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - cargo.env("LLVM_CONFIG", &llvm_config); - - // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script - // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by - // whitespace. - // - // For example: - // - on windows, when `clang-cl` is used with instrumentation, we need to manually add - // clang's runtime library resource directory so that the profiler runtime library can be - // found. This is to avoid the linker errors about undefined references to - // `__llvm_profile_instrument_memop` when linking `rustc_driver`. - let mut llvm_linker_flags = String::new(); - if builder.config.llvm_profile_generate && target.is_msvc() { - if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { - // Add clang's runtime library directory to the search path - let clang_rt_dir = get_clang_cl_resource_dir(builder, clang_cl_path); - llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); - } - } - - // The config can also specify its own llvm linker flags. - if let Some(ref s) = builder.config.llvm_ldflags { - if !llvm_linker_flags.is_empty() { - llvm_linker_flags.push(' '); - } - llvm_linker_flags.push_str(s); - } - - // Set the linker flags via the env var that `rustc_llvm`'s build script will read. - if !llvm_linker_flags.is_empty() { - cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); - } - - // Building with a static libstdc++ is only supported on linux right now, - // not for MSVC or macOS - if builder.config.llvm_static_stdcpp - && !target.contains("freebsd") - && !target.is_msvc() - && !target.contains("apple") - && !target.contains("solaris") - { - let file = compiler_file( - builder, - &builder.cxx(target).unwrap(), - target, - CLang::Cxx, - "libstdc++.a", - ); - cargo.env("LLVM_STATIC_STDCPP", file); - } - if builder.llvm_link_shared() { - cargo.env("LLVM_LINK_SHARED", "1"); - } - if builder.config.llvm_use_libcxx { - cargo.env("LLVM_USE_LIBCXX", "1"); - } - if builder.config.llvm_assertions { - cargo.env("LLVM_ASSERTIONS", "1"); - } -} - -/// `RustcLink` copies all of the rlibs from the rustc build into the previous stage's sysroot. -/// This is necessary for tools using `rustc_private`, where the previous compiler will build -/// a tool against the next compiler. -/// To build a tool against a compiler, the rlibs of that compiler that it links against -/// must be in the sysroot of the compiler that's doing the compiling. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RustcLink { - /// The compiler whose rlibs we are copying around. - pub compiler: Compiler, - /// This is the compiler into whose sysroot we want to copy the rlibs into. - pub previous_stage_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Vec, -} - -impl RustcLink { - fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - previous_stage_compiler: rustc.compiler, - target: rustc.target, - crates: rustc.crates, - } - } -} - -impl Step for RustcLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Same as `std_link`, only for librustc - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let previous_stage_compiler = self.previous_stage_compiler; - let target = self.target; - add_to_sysroot( - builder, - &builder.sysroot_target_libdir(previous_stage_compiler, target), - &builder.sysroot_target_libdir(previous_stage_compiler, compiler.host), - &librustc_stamp(builder, compiler, target), - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub compiler: Compiler, - pub backend: String, -} - -fn needs_codegen_config(run: &RunConfig<'_>) -> bool { - let mut needs_codegen_cfg = false; - for path_set in &run.paths { - needs_codegen_cfg = match path_set { - PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), - PathSet::Suite(suite) => is_codegen_cfg_needed(suite, run), - } - } - needs_codegen_cfg -} +// --- Remaining top-level items --- pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; - -fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { - let path = path.path.to_str().unwrap(); - - let is_explicitly_called = |p| -> bool { run.builder.paths.contains(p) }; - let should_enforce = run.builder.kind == Kind::Dist || run.builder.kind == Kind::Install; - - if path.contains(CODEGEN_BACKEND_PREFIX) { - let mut needs_codegen_backend_config = true; - for backend in run.builder.config.codegen_backends(run.target) { - if path.ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + backend)) { - needs_codegen_backend_config = false; - } - } - if (is_explicitly_called(&PathBuf::from(path)) || should_enforce) - && needs_codegen_backend_config - { - run.builder.info( - "WARNING: no codegen-backends config matched the requested path to build a codegen backend. \ - HELP: add backend to codegen-backends in config.toml.", - ); - return true; - } - } - - false -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - /// Only the backends specified in the `codegen-backends` entry of `config.toml` are built. - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - if needs_codegen_config(&run) { - return; - } - - for backend in run.builder.config.codegen_backends(run.target) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - run.builder.ensure(CodegenBackend { - target: run.target, - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - backend: backend.clone(), - }); - } - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info( - "WARNING: Using a potentially old codegen backend. \ - This may not behave well.", - ); - // Codegen backends are linked separately from this step today, so we don't do - // anything here. - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); - return; - } - - let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Codegen, - SourceType::InTree, - target, - Kind::Build, - ); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let tmp_stamp = out_dir.join(".tmp.stamp"); - - let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); - if builder.config.dry_run() { - return; - } - let mut files = files.into_iter().filter(|f| { - let filename = f.file_name().unwrap().to_str().unwrap(); - is_dylib(f) && filename.contains("rustc_codegen_") - }); - let codegen_backend = match files.next() { - Some(f) => f, - None => panic!("no dylibs built for codegen backend?"), - }; - if let Some(f) = files.next() { - panic!( - "codegen backend built two dylibs:\n{}\n{}", - codegen_backend.display(), - f.display() - ); - } - let stamp = codegen_backend_stamp(builder, compiler, target, &backend); - let codegen_backend = codegen_backend.to_str().unwrap(); - t!(fs::write(stamp, codegen_backend)); - } -} - -/// Creates the `codegen-backends` folder for a compiler that's about to be -/// assembled as a complete compiler. -/// -/// This will take the codegen artifacts produced by `compiler` and link them -/// into an appropriate location for `target_compiler` to be a functional -/// compiler. -fn copy_codegen_backends_to_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - target_compiler: Compiler, -) { - let target = target_compiler.host; - - // Note that this step is different than all the other `*Link` steps in - // that it's not assembling a bunch of libraries but rather is primarily - // moving the codegen backend into place. The codegen backend of rustc is - // not linked into the main compiler by default but is rather dynamically - // selected at runtime for inclusion. - // - // Here we're looking for the output dylib of the `CodegenBackend` step and - // we're copying that into the `codegen-backends` folder. - let dst = builder.sysroot_codegen_backends(target_compiler); - t!(fs::create_dir_all(&dst), dst); - - if builder.config.dry_run() { - return; - } - - for backend in builder.config.codegen_backends(target) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - let stamp = codegen_backend_stamp(builder, compiler, target, backend); - let dylib = t!(fs::read_to_string(&stamp)); - let file = Path::new(&dylib); - let filename = file.file_name().unwrap().to_str().unwrap(); - // change `librustc_codegen_cranelift-xxxxxx.so` to - // `librustc_codegen_cranelift-release.so` - let target_filename = { - let dash = filename.find('-').unwrap(); - let dot = filename.find('.').unwrap(); - format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) - }; - builder.copy_link(file, &dst.join(target_filename)); - } -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -pub fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: &str, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}.stamp")) -} - -pub fn compiler_file( - builder: &Builder<'_>, - compiler: &Path, - target: TargetSelection, - c: CLang, - file: &str, -) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::new(); - } - let mut cmd = command(compiler); - cmd.args(builder.cflags(target, GitRepo::Rustc, c)); - cmd.arg(format!("-print-file-name={file}")); - let out = cmd.run_capture_stdout(builder).stdout(); - PathBuf::from(out.trim()) -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Sysroot { - pub compiler: Compiler, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl Sysroot { - pub(crate) fn new(compiler: Compiler) -> Self { - Sysroot { compiler, force_recompile: false } - } -} - -impl Step for Sysroot { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Returns the sysroot that `compiler` is supposed to use. - /// For the stage0 compiler, this is stage0-sysroot (because of the initial std build). - /// For all other stages, it's the same stage directory that the compiler lives in. - fn run(self, builder: &Builder<'_>) -> PathBuf { - let compiler = self.compiler; - let host_dir = builder.out.join(compiler.host); - - let sysroot_dir = |stage| { - if stage == 0 { - host_dir.join("stage0-sysroot") - } else if self.force_recompile && stage == compiler.stage { - host_dir.join(format!("stage{stage}-test-sysroot")) - } else if builder.download_rustc() && compiler.stage != builder.top_stage { - host_dir.join("ci-rustc-sysroot") - } else { - host_dir.join(format!("stage{}", stage)) - } - }; - let sysroot = sysroot_dir(compiler.stage); - - builder - .verbose(|| println!("Removing sysroot {} to avoid caching bugs", sysroot.display())); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - - // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 - // compiler relies on more recent version of LLVM than the beta compiler, it may not - // be able to locate the correct LLVM in the sysroot. This situation typically occurs - // when we upgrade LLVM version while the beta compiler continues to use an older version. - // - // Make sure to add the correct version of LLVM into the stage0 sysroot. - if compiler.stage == 0 { - dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); - } - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() && compiler.stage != 0 { - assert_eq!( - builder.config.build, compiler.host, - "Cross-compiling is not yet supported with `download-rustc`", - ); - - // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. - for stage in 0..=2 { - if stage != compiler.stage { - let dir = sysroot_dir(stage); - if !dir.ends_with("ci-rustc-sysroot") { - let _ = fs::remove_dir_all(dir); - } - } - } - - // Copy the compiler into the correct sysroot. - // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. - // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. - // There are a few quirks of bootstrap that interact to make this reliable: - // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This - // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to - // fail because of duplicate metadata. - // 2. The sysroot is deleted and recreated between each invocation, so running `x test - // ui-fulldeps && x test ui` can't cause failures. - let mut filtered_files = Vec::new(); - let mut add_filtered_files = |suffix, contents| { - for path in contents { - let path = Path::new(&path); - if path.parent().map_or(false, |parent| parent.ends_with(suffix)) { - filtered_files.push(path.file_name().unwrap().to_owned()); - } - } - }; - let suffix = format!("lib/rustlib/{}/lib", compiler.host); - add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); - // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the - // newly compiled std, not the downloaded std. - add_filtered_files("lib", builder.config.ci_rust_std_contents()); - - let filtered_extensions = [ - OsStr::new("rmeta"), - OsStr::new("rlib"), - // FIXME: this is wrong when compiler.host != build, but we don't support that today - OsStr::new(std::env::consts::DLL_EXTENSION), - ]; - let ci_rustc_dir = builder.config.ci_rustc_dir(); - builder.cp_link_filtered(&ci_rustc_dir, &sysroot, &|path| { - if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { - return true; - } - if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { - return true; - } - if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { - builder.verbose_than(1, || println!("ignoring {}", path.display())); - false - } else { - true - } - }); - } - - // Symlink the source root into the same location inside the sysroot, - // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), - // so that any tools relying on `rust-src` also work for local builds, - // and also for translating the virtual `/rustc/$hash` back to the real - // directory (for running tests with `rust.remap-debuginfo = true`). - let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); - let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); - if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { - eprintln!( - "ERROR: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_src_rust.display(), - builder.src.display(), - e, - ); - if builder.config.rust_remap_debuginfo { - eprintln!( - "ERROR: some `tests/ui` tests will fail when lacking `{}`", - sysroot_lib_rustlib_src_rust.display(), - ); - } - build_helper::exit!(1); - } - - // rustc-src component is already part of CI rustc's sysroot - if !builder.download_rustc() { - let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); - let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); - if let Err(e) = - symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) - { - eprintln!( - "ERROR: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_rustcsrc_rust.display(), - builder.src.display(), - e, - ); - build_helper::exit!(1); - } - } - - sysroot - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Assemble { - /// The compiler which we will produce in this step. Assemble itself will - /// take care of ensuring that the necessary prerequisites to do so exist, - /// that is, this target can be a stage2 compiler and Assemble will build - /// previous stages for you. - pub target_compiler: Compiler, -} - -impl Step for Assemble { - type Output = Compiler; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Assemble { - target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), - }); - } - - /// Prepare a new compiler from the artifacts in `stage` - /// - /// This will assemble a compiler in `build/$host/stage$stage`. The compiler - /// must have been previously produced by the `stage - 1` builder.build - /// compiler. - fn run(self, builder: &Builder<'_>) -> Compiler { - let target_compiler = self.target_compiler; - - if target_compiler.stage == 0 { - assert_eq!( - builder.config.build, target_compiler.host, - "Cannot obtain compiler for non-native build triple at stage 0" - ); - // The stage 0 compiler for the build triple is always pre-built. - return target_compiler; - } - - // We prepend this bin directory to the user PATH when linking Rust binaries. To - // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. - let libdir = builder.sysroot_target_libdir(target_compiler, target_compiler.host); - let libdir_bin = libdir.parent().unwrap().join("bin"); - t!(fs::create_dir_all(&libdir_bin)); - - if builder.config.llvm_enabled(target_compiler.host) { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: target_compiler.host }); - if !builder.config.dry_run() && builder.config.llvm_tools_enabled { - let llvm_bin_dir = - command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout(); - let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); - - // Since we've already built the LLVM tools, install them to the sysroot. - // This is the equivalent of installing the `llvm-tools-preview` component via - // rustup, and lets developers use a locally built toolchain to - // build projects that expect llvm tools to be present in the sysroot - // (e.g. the `bootimage` crate). - for tool in LLVM_TOOLS { - let tool_exe = exe(tool, target_compiler.host); - let src_path = llvm_bin_dir.join(&tool_exe); - // When using `download-ci-llvm`, some of the tools - // may not exist, so skip trying to copy them. - if src_path.exists() { - // There is a chance that these tools are being installed from an external LLVM. - // Use `Builder::resolve_symlink_and_copy` instead of `Builder::copy_link` to ensure - // we are copying the original file not the symlinked path, which causes issues for - // tarball distribution. - // - // See https://github.com/rust-lang/rust/issues/135554. - builder.resolve_symlink_and_copy(&src_path, &libdir_bin.join(&tool_exe)); - } - } - } - } - - let maybe_install_llvm_bitcode_linker = |compiler| { - if builder.config.llvm_bitcode_linker_enabled { - let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { - compiler, - target: target_compiler.host, - extra_features: vec![], - }); - let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); - builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); - } - }; - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() { - builder.ensure(Std::new(target_compiler, target_compiler.host)); - let sysroot = - builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); - // Ensure that `libLLVM.so` ends up in the newly created target directory, - // so that tools using `rustc_private` can use it. - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - // Lower stages use `ci-rustc-sysroot`, not stageN - if target_compiler.stage == builder.top_stage { - builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); - } - - maybe_install_llvm_bitcode_linker(target_compiler); - - return target_compiler; - } - - // Get the compiler that we'll use to bootstrap ourselves. - // - // Note that this is where the recursive nature of the bootstrap - // happens, as this will request the previous stage's compiler on - // downwards to stage 0. - // - // Also note that we're building a compiler for the host platform. We - // only assume that we can run `build` artifacts, which means that to - // produce some other architecture compiler we need to start from - // `build` to get there. - // - // FIXME: It may be faster if we build just a stage 1 compiler and then - // use that to bootstrap this compiler forward. - let mut build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // Build enzyme - let enzyme_install = if builder.config.llvm_enzyme { - Some(builder.ensure(llvm::Enzyme { target: build_compiler.host })) - } else { - None - }; - - if let Some(enzyme_install) = enzyme_install { - let lib_ext = std::env::consts::DLL_EXTENSION; - let src_lib = enzyme_install.join("build/Enzyme/libEnzyme-19").with_extension(lib_ext); - let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); - let target_libdir = - builder.sysroot_target_libdir(target_compiler, target_compiler.host); - let dst_lib = libdir.join("libEnzyme-19").with_extension(lib_ext); - let target_dst_lib = target_libdir.join("libEnzyme-19").with_extension(lib_ext); - builder.copy_link(&src_lib, &dst_lib); - builder.copy_link(&src_lib, &target_dst_lib); - } - - // Build the libraries for this compiler to link to (i.e., the libraries - // it uses at runtime). NOTE: Crates the target compiler compiles don't - // link to these. (FIXME: Is that correct? It seems to be correct most - // of the time but I think we do link to these for stage2/bin compilers - // when not performing a full bootstrap). - let actual_stage = builder.ensure(Rustc::new(build_compiler, target_compiler.host)); - // Current build_compiler.stage might be uplifted instead of being built; so update it - // to not fail while linking the artifacts. - build_compiler.stage = actual_stage; - - for backend in builder.config.codegen_backends(target_compiler.host) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - builder.ensure(CodegenBackend { - compiler: build_compiler, - target: target_compiler.host, - backend: backend.clone(), - }); - } - - let lld_install = if builder.config.lld_enabled { - Some(builder.ensure(llvm::Lld { target: target_compiler.host })) - } else { - None - }; - - let stage = target_compiler.stage; - let host = target_compiler.host; - let (host_info, dir_name) = if build_compiler.host == host { - ("".into(), "host".into()) - } else { - (format!(" ({host})"), host.to_string()) - }; - // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since - // sysroots can temporarily be empty until we put the compiler inside. However, - // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any - // ambiguity. - let msg = format!( - "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" - ); - builder.info(&msg); - - // Link in all dylibs to the libdir - let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); - let proc_macros = builder - .read_stamp_file(&stamp) - .into_iter() - .filter_map(|(path, dependency_type)| { - if dependency_type == DependencyType::Host { - Some(path.file_name().unwrap().to_owned().into_string().unwrap()) - } else { - None - } - }) - .collect::>(); - - let sysroot = builder.sysroot(target_compiler); - let rustc_libdir = builder.rustc_libdir(target_compiler); - t!(fs::create_dir_all(&rustc_libdir)); - let src_libdir = builder.sysroot_target_libdir(build_compiler, host); - for f in builder.read_dir(&src_libdir) { - let filename = f.file_name().into_string().unwrap(); - - let is_proc_macro = proc_macros.contains(&filename); - let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename); - - // If we link statically to stdlib, do not copy the libstd dynamic library file - // FIXME: Also do this for Windows once incremental post-optimization stage0 tests - // work without std.dll (see https://github.com/rust-lang/rust/pull/131188). - let can_be_rustc_dynamic_dep = if builder - .link_std_into_rustc_driver(target_compiler.host) - && !target_compiler.host.is_windows() - { - let is_std = filename.starts_with("std-") || filename.starts_with("libstd-"); - !is_std - } else { - true - }; - - if is_dylib_or_debug && can_be_rustc_dynamic_dep && !is_proc_macro { - builder.copy_link(&f.path(), &rustc_libdir.join(&filename)); - } - } - - copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - - if let Some(lld_install) = lld_install { - let src_exe = exe("lld", target_compiler.host); - let dst_exe = exe("rust-lld", target_compiler.host); - builder.copy_link(&lld_install.join("bin").join(src_exe), &libdir_bin.join(dst_exe)); - let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir_all(&self_contained_lld_dir)); - let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { - compiler: build_compiler, - target: target_compiler.host, - }); - for name in crate::LLD_FILE_NAMES { - builder.copy_link( - &lld_wrapper_exe, - &self_contained_lld_dir.join(exe(name, target_compiler.host)), - ); - } - } - - if builder.config.llvm_enabled(target_compiler.host) && builder.config.llvm_tools_enabled { - // `llvm-strip` is used by rustc, which is actually just a symlink to `llvm-objcopy`, so - // copy and rename `llvm-objcopy`. - // - // But only do so if llvm-tools are enabled, as bootstrap compiler might not contain any - // LLVM tools, e.g. for cg_clif. - // See . - let src_exe = exe("llvm-objcopy", target_compiler.host); - let dst_exe = exe("rust-objcopy", target_compiler.host); - builder.copy_link(&libdir_bin.join(src_exe), &libdir_bin.join(dst_exe)); - } - - // In addition to `rust-lld` also install `wasm-component-ld` when - // LLD is enabled. This is a relatively small binary that primarily - // delegates to the `rust-lld` binary for linking and then runs - // logic to create the final binary. This is used by the - // `wasm32-wasip2` target of Rust. - if builder.tool_enabled("wasm-component-ld") { - let wasm_component_ld_exe = - builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, - target: target_compiler.host, - }); - builder.copy_link( - &wasm_component_ld_exe, - &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), - ); - } - - maybe_install_llvm_bitcode_linker(build_compiler); - - // Ensure that `libLLVM.so` ends up in the newly build compiler directory, - // so that it can be found when the newly built `rustc` is run. - dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - - // Link the compiler binary itself into place - let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); - let rustc = out_dir.join(exe("rustc-main", host)); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(bindir)); - let compiler = builder.rustc(target_compiler); - builder.copy_link(&rustc, &compiler); - - target_compiler - } -} - -/// Link some files into a rustc sysroot. -/// -/// For a particular stage this will link the file listed in `stamp` into the -/// `sysroot_dst` provided. -pub fn add_to_sysroot( - builder: &Builder<'_>, - sysroot_dst: &Path, - sysroot_host_dst: &Path, - stamp: &Path, -) { - let self_contained_dst = &sysroot_dst.join("self-contained"); - t!(fs::create_dir_all(sysroot_dst)); - t!(fs::create_dir_all(sysroot_host_dst)); - t!(fs::create_dir_all(self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - let dst = match dependency_type { - DependencyType::Host => sysroot_host_dst, - DependencyType::Target => sysroot_dst, - DependencyType::TargetSelfContained => self_contained_dst, - }; - builder.copy_link(&path, &dst.join(path.file_name().unwrap())); - } -} - -pub fn run_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - stamp: &Path, - additional_target_deps: Vec<(PathBuf, DependencyType)>, - is_check: bool, - rlib_only_metadata: bool, -) -> Vec { - // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.parent().unwrap(); - // `target_deps_dir` looks like $dir/$target/release/deps - let target_deps_dir = target_root_dir.join("deps"); - // `host_root_dir` looks like $dir/release - let host_root_dir = target_root_dir - .parent() - .unwrap() // chop off `release` - .parent() - .unwrap() // chop off `$target` - .join(target_root_dir.file_name().unwrap()); - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let mut deps = Vec::new(); - let mut toplevel = Vec::new(); - let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { - let (filenames, crate_types) = match msg { - CargoMessage::CompilerArtifact { - filenames, - target: CargoTarget { crate_types }, - .. - } => (filenames, crate_types), - _ => return, - }; - for filename in filenames { - // Skip files like executables - let mut keep = false; - if filename.ends_with(".lib") - || filename.ends_with(".a") - || is_debug_info(&filename) - || is_dylib(Path::new(&*filename)) - { - // Always keep native libraries, rust dylibs and debuginfo - keep = true; - } - if is_check && filename.ends_with(".rmeta") { - // During check builds we need to keep crate metadata - keep = true; - } else if rlib_only_metadata { - if filename.contains("jemalloc_sys") - || filename.contains("rustc_smir") - || filename.contains("stable_mir") - { - // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, - // so we need to distribute them as rlib to be able to use them. - keep |= filename.ends_with(".rlib"); - } else { - // Distribute the rest of the rustc crates as rmeta files only to reduce - // the tarball sizes by about 50%. The object files are linked into - // librustc_driver.so, so it is still possible to link against them. - keep |= filename.ends_with(".rmeta"); - } - } else { - // In all other cases keep all rlibs - keep |= filename.ends_with(".rlib"); - } - - if !keep { - continue; - } - - let filename = Path::new(&*filename); - - // If this was an output file in the "host dir" we don't actually - // worry about it, it's not relevant for us - if filename.starts_with(&host_root_dir) { - // Unless it's a proc macro used in the compiler - if crate_types.iter().any(|t| t == "proc-macro") { - deps.push((filename.to_path_buf(), DependencyType::Host)); - } - continue; - } - - // If this was output in the `deps` dir then this is a precise file - // name (hash included) so we start tracking it. - if filename.starts_with(&target_deps_dir) { - deps.push((filename.to_path_buf(), DependencyType::Target)); - continue; - } - - // Otherwise this was a "top level artifact" which right now doesn't - // have a hash in the name, but there's a version of this file in - // the `deps` folder which *does* have a hash in the name. That's - // the one we'll want to we'll probe for it later. - // - // We do not use `Path::file_stem` or `Path::extension` here, - // because some generated files may have multiple extensions e.g. - // `std-.dll.lib` on Windows. The aforementioned methods only - // split the file name by the last extension (`.lib`) while we need - // to split by all extensions (`.dll.lib`). - let expected_len = t!(filename.metadata()).len(); - let filename = filename.file_name().unwrap().to_str().unwrap(); - let mut parts = filename.splitn(2, '.'); - let file_stem = parts.next().unwrap().to_owned(); - let extension = parts.next().unwrap().to_owned(); - - toplevel.push((file_stem, extension, expected_len)); - } - }); - - if !ok { - crate::exit!(1); - } - - if builder.config.dry_run() { - return Vec::new(); - } - - // Ok now we need to actually find all the files listed in `toplevel`. We've - // got a list of prefix/extensions and we basically just need to find the - // most recent file in the `deps` folder corresponding to each one. - let contents = t!(target_deps_dir.read_dir()) - .map(|e| t!(e)) - .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) - .collect::>(); - for (prefix, extension, expected_len) in toplevel { - let candidates = contents.iter().filter(|&(_, filename, meta)| { - meta.len() == expected_len - && filename - .strip_prefix(&prefix[..]) - .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) - .unwrap_or(false) - }); - let max = candidates.max_by_key(|&(_, _, metadata)| { - metadata.modified().expect("mtime should be available on all relevant OSes") - }); - let path_to_add = match max { - Some(triple) => triple.0.to_str().unwrap(), - None => panic!("no output generated for {prefix:?} {extension:?}"), - }; - if is_dylib(Path::new(path_to_add)) { - let candidate = format!("{path_to_add}.lib"); - let candidate = PathBuf::from(candidate); - if candidate.exists() { - deps.push((candidate, DependencyType::Target)); - } - } - deps.push((path_to_add.into(), DependencyType::Target)); - } - - deps.extend(additional_target_deps); - deps.sort(); - let mut new_contents = Vec::new(); - for (dep, dependency_type) in deps.iter() { - new_contents.extend(match *dependency_type { - DependencyType::Host => b"h", - DependencyType::Target => b"t", - DependencyType::TargetSelfContained => b"s", - }); - new_contents.extend(dep.to_str().unwrap().as_bytes()); - new_contents.extend(b"\0"); - } - t!(fs::write(stamp, &new_contents)); - deps.into_iter().map(|(d, _)| d).collect() -} - -pub fn stream_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - cb: &mut dyn FnMut(CargoMessage<'_>), -) -> bool { - let mut cmd = cargo.into_cmd(); - let cargo = cmd.as_command_mut(); - // Instruct Cargo to give us json messages on stdout, critically leaving - // stderr as piped so we can get those pretty colors. - let mut message_format = if builder.config.json_output { - String::from("json") - } else { - String::from("json-render-diagnostics") - }; - if let Some(s) = &builder.config.rustc_error_format { - message_format.push_str(",json-diagnostic-"); - message_format.push_str(s); - } - cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); - - for arg in tail_args { - cargo.arg(arg); - } - - builder.verbose(|| println!("running: {cargo:?}")); - - if builder.config.dry_run() { - return true; - } - - let mut child = match cargo.spawn() { - Ok(child) => child, - Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), - }; - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let stdout = BufReader::new(child.stdout.take().unwrap()); - for line in stdout.lines() { - let line = t!(line); - match serde_json::from_str::>(&line) { - Ok(msg) => { - if builder.config.json_output { - // Forward JSON to stdout. - println!("{line}"); - } - cb(msg) - } - // If this was informational, just print it out and continue - Err(_) => println!("{line}"), - } - } - - // Make sure Cargo actually succeeded after we read all of its stdout. - let status = t!(child.wait()); - if builder.is_verbose() && !status.success() { - eprintln!( - "command did not execute successfully: {cargo:?}\n\ - expected success, got: {status}" - ); - } - status.success() -} - -#[derive(Deserialize)] -pub struct CargoTarget<'a> { - crate_types: Vec>, -} - -#[derive(Deserialize)] -#[serde(tag = "reason", rename_all = "kebab-case")] -pub enum CargoMessage<'a> { - CompilerArtifact { filenames: Vec>, target: CargoTarget<'a> }, - BuildScriptExecuted, - BuildFinished, -} - -pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { - // FIXME: to make things simpler for now, limit this to the host and target where we know - // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not - // cross-compiling. Expand this to other appropriate targets in the future. - if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { - return; - } - - let previous_mtime = t!(t!(path.metadata()).modified()); - command("strip").arg("--strip-debug").arg(path).run_capture(builder); - - let file = t!(fs::File::open(path)); - - // After running `strip`, we have to set the file modification time to what it was before, - // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time - // bootstrap is invoked. - // - // An example of this is if we run this on librustc_driver.so. In the first invocation: - // - Cargo will build librustc_driver.so (mtime of 1) - // - Cargo will build rustc-main (mtime of 2) - // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). - // - // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so - // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause - // everything else (standard library, future stages...) to be rebuilt. - t!(file.set_modified(previous_mtime)); -} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/add_to_sysroot.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/add_to_sysroot.rs new file mode 100644 index 00000000..f28c2d95 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/add_to_sysroot.rs @@ -0,0 +1,26 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `add_to_sysroot` from `compile.rs`. + +/// Link some files into a rustc sysroot. +/// +/// For a particular stage this will link the file listed in `stamp` into the +/// `sysroot_dst` provided. +pub fn add_to_sysroot( + builder: &Builder<'_>, + sysroot_dst: &Path, + sysroot_host_dst: &Path, + stamp: &Path, +) { + let self_contained_dst = &sysroot_dst.join("self-contained"); + t!(fs::create_dir_all(sysroot_dst)); + t!(fs::create_dir_all(sysroot_host_dst)); + t!(fs::create_dir_all(self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + let dst = match dependency_type { + DependencyType::Host => sysroot_host_dst, + DependencyType::Target => sysroot_dst, + DependencyType::TargetSelfContained => self_contained_dst, + }; + builder.copy_link(&path, &dst.join(path.file_name().unwrap())); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_sign_file.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_sign_file.rs new file mode 100644 index 00000000..e7d5cde0 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_sign_file.rs @@ -0,0 +1,11 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `apple_darwin_sign_file` from `compile.rs`. + +fn apple_darwin_sign_file(builder: &Builder<'_>, file_path: &Path) { + command("codesign") + .arg("-f") // Force to rewrite the existing signature + .arg("-s") + .arg("-") + .arg(file_path) + .run(builder); +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_update_library_name.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_update_library_name.rs new file mode 100644 index 00000000..fd662ce7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/apple_darwin_update_library_name.rs @@ -0,0 +1,6 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `apple_darwin_update_library_name` from `compile.rs`. + +fn apple_darwin_update_library_name(builder: &Builder<'_>, library_path: &Path, new_name: &str) { + command("install_name_tool").arg("-id").arg(new_name).arg(library_path).run(builder); +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/assemble.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/assemble.rs new file mode 100644 index 00000000..92ebd58d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/assemble.rs @@ -0,0 +1,297 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Assemble` from `compile.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Assemble { + /// The compiler which we will produce in this step. Assemble itself will + /// take care of ensuring that the necessary prerequisites to do so exist, + /// that is, this target can be a stage2 compiler and Assemble will build + /// previous stages for you. + pub target_compiler: Compiler, +} + +impl Step for Assemble { + type Output = Compiler; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Assemble { + target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), + }); + } + + /// Prepare a new compiler from the artifacts in `stage` + /// + /// This will assemble a compiler in `build/$host/stage$stage`. The compiler + /// must have been previously produced by the `stage - 1` builder.build + /// compiler. + fn run(self, builder: &Builder<'_>) -> Compiler { + let target_compiler = self.target_compiler; + + if target_compiler.stage == 0 { + assert_eq!( + builder.config.build, target_compiler.host, + "Cannot obtain compiler for non-native build triple at stage 0" + ); + // The stage 0 compiler for the build triple is always pre-built. + return target_compiler; + } + + // We prepend this bin directory to the user PATH when linking Rust binaries. To + // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. + let libdir = builder.sysroot_target_libdir(target_compiler, target_compiler.host); + let libdir_bin = libdir.parent().unwrap().join("bin"); + t!(fs::create_dir_all(&libdir_bin)); + + if builder.config.llvm_enabled(target_compiler.host) { + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: target_compiler.host }); + if !builder.config.dry_run && builder.config.llvm_tools_enabled { + let llvm_bin_dir = + command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout(); + let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); + + // Since we've already built the LLVM tools, install them to the sysroot. + // This is the equivalent of installing the `llvm-tools-preview` component via + // rustup, and lets developers use a locally built toolchain to + // build projects that expect llvm tools to be present in the sysroot + // (e.g. the `bootimage` crate). + for tool in LLVM_TOOLS { + let tool_exe = exe(tool, target_compiler.host); + let src_path = llvm_bin_dir.join(&tool_exe); + // When using `download-ci-llvm`, some of the tools + // may not exist, so skip trying to copy them. + if src_path.exists() { + // There is a chance that these tools are being installed from an external LLVM. + // Use `Builder::resolve_symlink_and_copy` instead of `Builder::copy_link` to ensure + // we are copying the original file not the symlinked path, which causes issues for + // tarball distribution. + // + // See https://github.com/rust-lang/rust/issues/135554. + builder.resolve_symlink_and_copy(&src_path, &libdir_bin.join(&tool_exe)); + } + } + } + } + + let maybe_install_llvm_bitcode_linker = |compiler| { + if builder.config.llvm_bitcode_linker_enabled { + let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { + compiler, + target: target_compiler.host, + extra_features: vec![], + }); + let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); + builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); + } + }; + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() { + builder.ensure(Std::new(target_compiler, target_compiler.host)); + let sysroot = + builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); + // Ensure that `libLLVM.so` ends up in the newly created target directory, + // so that tools using `rustc_private` can use it. + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + // Lower stages use `ci-rustc-sysroot`, not stageN + if target_compiler.stage == builder.top_stage { + builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); + } + + maybe_install_llvm_bitcode_linker(target_compiler); + + return target_compiler; + } + + // Get the compiler that we'll use to bootstrap ourselves. + // + // Note that this is where the recursive nature of the bootstrap + // happens, as this will request the previous stage's compiler on + // downwards to stage 0. + // + // Also note that we're building a compiler for the host platform. We + // only assume that we can run `build` artifacts, which means that to + // produce some other architecture compiler we need to start from + // `build` to get there. + // + // FIXME: It may be faster if we build just a stage 1 compiler and then + // use that to bootstrap this compiler forward. + let mut build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // Build enzyme + let enzyme_install = if builder.config.llvm_enzyme { + Some(builder.ensure(llvm::Enzyme { target: build_compiler.host })) + } else { + None + }; + + if let Some(enzyme_install) = enzyme_install { + let lib_ext = std::env::consts::DLL_EXTENSION; + let src_lib = enzyme_install.join("build/Enzyme/libEnzyme-19").with_extension(lib_ext); + let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); + let target_libdir = + builder.sysroot_target_libdir(target_compiler, target_compiler.host); + let dst_lib = libdir.join("libEnzyme-19").with_extension(lib_ext); + let target_dst_lib = target_libdir.join("libEnzyme-19").with_extension(lib_ext); + builder.copy_link(&src_lib, &dst_lib); + builder.copy_link(&src_lib, &target_dst_lib); + } + + // Build the libraries for this compiler to link to (i.e., the libraries + // it uses at runtime). NOTE: Crates the target compiler compiles don't + // link to these. (FIXME: Is that correct? It seems to be correct most + // of the time but I think we do link to these for stage2/bin compilers + // when not performing a full bootstrap). + let actual_stage = builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + // Current build_compiler.stage might be uplifted instead of being built; so update it + // to not fail while linking the artifacts. + build_compiler.stage = actual_stage; + + for backend in builder.config.codegen_backends(target_compiler.host) { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + builder.ensure(CodegenBackend { + compiler: build_compiler, + target: target_compiler.host, + backend: backend.clone(), + }); + } + + let lld_install = if builder.config.lld_enabled { + Some(builder.ensure(llvm::Lld { target: target_compiler.host })) + } else { + None + }; + + let stage = target_compiler.stage; + let host = target_compiler.host; + let (host_info, dir_name) = if build_compiler.host == host { + ("".into(), "host".into()) + } else { + (format!(" ({host})"), host.to_string()) + }; + // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since + // sysroots can temporarily be empty until we put the compiler inside. However, + // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any + // ambiguity. + let msg = format!( + "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" + ); + builder.info(&msg); + + // Link in all dylibs to the libdir + let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let proc_macros = builder + .read_stamp_file(&stamp) + .into_iter() + .filter_map(|(path, dependency_type)| { + if dependency_type == DependencyType::Host { + Some(path.file_name().unwrap().to_owned().into_string().unwrap()) + } else { + None + } + }) + .collect::>(); + + let sysroot = builder.sysroot(target_compiler); + let rustc_libdir = builder.rustc_libdir(target_compiler); + t!(fs::create_dir_all(&rustc_libdir)); + let src_libdir = builder.sysroot_target_libdir(build_compiler, host); + for f in builder.read_dir(&src_libdir) { + let filename = f.file_name().into_string().unwrap(); + + let is_proc_macro = proc_macros.contains(&filename); + let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename); + + // If we link statically to stdlib, do not copy the libstd dynamic library file + // FIXME: Also do this for Windows once incremental post-optimization stage0 tests + // work without std.dll (see https://github.com/rust-lang/rust/pull/131188). + let can_be_rustc_dynamic_dep = if builder + .link_std_into_rustc_driver(target_compiler.host) + && !target_compiler.host.is_windows() + { + let is_std = filename.starts_with("std-") || filename.starts_with("libstd-"); + !is_std + } else { + true + }; + + if is_dylib_or_debug && can_be_rustc_dynamic_dep && !is_proc_macro { + builder.copy_link(&f.path(), &rustc_libdir.join(&filename)); + } + } + + copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + + if let Some(lld_install) = lld_install { + let src_exe = exe("lld", target_compiler.host); + let dst_exe = exe("rust-lld", target_compiler.host); + builder.copy_link(&lld_install.join("bin").join(src_exe), &libdir_bin.join(dst_exe)); + let self_contained_lld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir_all(&self_contained_lld_dir)); + let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { + compiler: build_compiler, + target: target_compiler.host, + }); + for name in crate::LLD_FILE_NAMES { + builder.copy_link( + &lld_wrapper_exe, + &self_contained_lld_dir.join(exe(name, target_compiler.host)), + ); + } + } + + if builder.config.llvm_enabled(target_compiler.host) && builder.config.llvm_tools_enabled { + // `llvm-strip` is used by rustc, which is actually just a symlink to `llvm-objcopy`, so + // copy and rename `llvm-objcopy`. + // + // But only do so if llvm-tools are enabled, as bootstrap compiler might not contain any + // LLVM tools, e.g. for cg_clif. + // See . + let src_exe = exe("llvm-objcopy", target_compiler.host); + let dst_exe = exe("rust-objcopy", target_compiler.host); + builder.copy_link(&libdir_bin.join(src_exe), &libdir_bin.join(dst_exe)); + } + + // In addition to `rust-lld` also install `wasm-component-ld` when + // LLD is enabled. This is a relatively small binary that primarily + // delegates to the `rust-lld` binary for linking and then runs + // logic to create the final binary. This is used by the + // `wasm32-wasip2` target of Rust. + if builder.tool_enabled("wasm-component-ld") { + let wasm_component_ld_exe = + builder.ensure(crate::core::build_steps::tool::WasmComponentLd { + compiler: build_compiler, + target: target_compiler.host, + }); + builder.copy_link( + &wasm_component_ld_exe, + &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), + ); + } + + maybe_install_llvm_bitcode_linker(build_compiler); + + // Ensure that `libLLVM.so` ends up in the newly build compiler directory, + // so that it can be found when the newly built `rustc` is run. + dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + + // Link the compiler binary itself into place + let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); + let rustc = out_dir.join(exe("rustc-main", host)); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(bindir)); + let compiler = builder.rustc(target_compiler); + builder.copy_link(&rustc, &compiler); + + target_compiler + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_message.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_message.rs new file mode 100644 index 00000000..4e09662a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_message.rs @@ -0,0 +1,10 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `CargoMessage` from `compile.rs`. + +#[derive(Deserialize)] +#[serde(tag = "reason", rename_all = "kebab-case")] +pub enum CargoMessage<'a> { + CompilerArtifact { filenames: Vec>, target: CargoTarget<'a> }, + BuildScriptExecuted, + BuildFinished, +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_target.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_target.rs new file mode 100644 index 00000000..f38e9244 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cargo_target.rs @@ -0,0 +1,7 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `CargoTarget` from `compile.rs`. + +#[derive(Deserialize)] +pub struct CargoTarget<'a> { + crate_types: Vec>, +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend.rs new file mode 100644 index 00000000..754617fc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend.rs @@ -0,0 +1,142 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `CodegenBackend` from `compile.rs`. + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub compiler: Compiler, + pub backend: String, +} + +fn needs_codegen_config(run: &RunConfig<'_>) -> bool { + let mut needs_codegen_cfg = false; + for path_set in &run.paths { + needs_codegen_cfg = match path_set { + PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), + PathSet::Suite(suite) => is_codegen_cfg_needed(suite, run), + } + } + needs_codegen_cfg +} + +pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; + +fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { + let path = path.path.to_str().unwrap(); + + let is_explicitly_called = |p| -> bool { run.builder.paths.contains(p) }; + let should_enforce = run.builder.kind == Kind::Dist || run.builder.kind == Kind::Install; + + if path.contains(CODEGEN_BACKEND_PREFIX) { + let mut needs_codegen_backend_config = true; + for backend in run.builder.config.codegen_backends(run.target) { + if path.ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + backend)) { + needs_codegen_backend_config = false; + } + } + if (is_explicitly_called(&PathBuf::from(path)) || should_enforce) + && needs_codegen_backend_config + { + run.builder.info( + include_str!("string_constants.txt"), + ); + return true; + } + } + + false +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + /// Only the backends specified in the `codegen-backends` entry of `config.toml` are built. + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + if needs_codegen_config(&run) { + return; + } + + for backend in run.builder.config.codegen_backends(run.target) { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + target: run.target, + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + backend: backend.clone(), + }); + } + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info( + include_str!("codegen_backend_warning_2.txt"), + ); + // Codegen backends are linked separately from this step today, so we don't do + // anything here. + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); + return; + } + + let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); + + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Codegen, + SourceType::InTree, + target, + Kind::Build, + ); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + let tmp_stamp = out_dir.join(".tmp.stamp"); + + let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); + if builder.config.dry_run { + return; + } + let mut files = files.into_iter().filter(|f| { + let filename = f.file_name().unwrap().to_str().unwrap(); + is_dylib(f) && filename.contains("rustc_codegen_") + }); + let codegen_backend = match files.next() { + Some(f) => f, + None => panic!("no dylibs built for codegen backend?"), + }; + if let Some(f) = files.next() { + panic!( + "codegen backend built two dylibs:\n{} +{}", + codegen_backend.display(), + f.display() + ); + } + let stamp = codegen_backend_stamp(builder, compiler, target, &backend); + let codegen_backend = codegen_backend.to_str().unwrap(); + t!(fs::write(stamp, codegen_backend)); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend_stamp.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend_stamp.rs new file mode 100644 index 00000000..1b3b7461 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/codegen_backend_stamp.rs @@ -0,0 +1,15 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `codegen_backend_stamp` from `compile.rs`. + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: &str, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{backend}.stamp")) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_file.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_file.rs new file mode 100644 index 00000000..22f49372 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_file.rs @@ -0,0 +1,19 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `compiler_file` from `compile.rs`. + +pub fn compiler_file( + builder: &Builder<'_>, + compiler: &Path, + target: TargetSelection, + c: CLang, + file: &str, +) -> PathBuf { + if builder.config.dry_run { + return PathBuf::new(); + } + let mut cmd = command(compiler); + cmd.args(builder.cflags(target, GitRepo::Rustc, c)); + cmd.arg(format!("-print-file-name={file}")); + let out = cmd.run_capture_stdout(builder).stdout(); + PathBuf::from(out.trim()) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_rt_for_profiler.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_rt_for_profiler.rs new file mode 100644 index 00000000..e178259d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/compiler_rt_for_profiler.rs @@ -0,0 +1,25 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `compiler_rt_for_profiler` from `compile.rs`. + +/// Tries to find LLVM's `compiler-rt` source directory, for building `library/profiler_builtins`. +/// +/// Normally it lives in the `src/llvm-project` submodule, but if we will be using a +/// downloaded copy of CI LLVM, then we try to use the `compiler-rt` sources from +/// there instead, which lets us avoid checking out the LLVM submodule. +fn compiler_rt_for_profiler(builder: &Builder<'_>) -> PathBuf { + // Try to use `compiler-rt` sources from downloaded CI LLVM, if possible. + if builder.config.llvm_from_ci { + // CI LLVM might not have been downloaded yet, so try to download it now. + builder.config.maybe_download_ci_llvm(); + let ci_llvm_compiler_rt = builder.config.ci_llvm_root().join("compiler-rt"); + if ci_llvm_compiler_rt.exists() { + return ci_llvm_compiler_rt; + } + } + + // Otherwise, fall back to requiring the LLVM submodule. + builder.require_submodule("src/llvm-project", { + Some("The `build.profiler` config option requires `compiler-rt` sources from LLVM.") + }); + builder.src.join("src/llvm-project/compiler-rt") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_and_stamp.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_and_stamp.rs new file mode 100644 index 00000000..5bd93b9b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_and_stamp.rs @@ -0,0 +1,16 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_and_stamp` from `compile.rs`. + +fn copy_and_stamp( + builder: &Builder<'_>, + libdir: &Path, + sourcedir: &Path, + name: &str, + target_deps: &mut Vec<(PathBuf, DependencyType)>, + dependency_type: DependencyType, +) { + let target = libdir.join(name); + builder.copy_link(&sourcedir.join(name), &target); + + target_deps.push((target, dependency_type)); +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_llvm_libunwind.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_llvm_libunwind.rs new file mode 100644 index 00000000..90ef91bc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_llvm_libunwind.rs @@ -0,0 +1,10 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_llvm_libunwind` from `compile.rs`. + +fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { + let libunwind_path = builder.ensure(llvm::Libunwind { target }); + let libunwind_source = libunwind_path.join("libunwind.a"); + let libunwind_target = libdir.join("libunwind.a"); + builder.copy_link(&libunwind_source, &libunwind_target); + libunwind_target +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_sanitizers.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_sanitizers.rs new file mode 100644 index 00000000..11888c2e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_sanitizers.rs @@ -0,0 +1,43 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_sanitizers` from `compile.rs`. + +/// Copies sanitizer runtime libraries into target libdir. +fn copy_sanitizers( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec { + let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); + + if builder.config.dry_run { + return Vec::new(); + } + + let mut target_deps = Vec::new(); + let libdir = builder.sysroot_target_libdir(*compiler, target); + + for runtime in &runtimes { + let dst = libdir.join(&runtime.name); + builder.copy_link(&runtime.path, &dst); + + // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for + // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do + // not list them here to rename and sign the runtime library. + if target == "x86_64-apple-darwin" + || target == "aarch64-apple-darwin" + || target == "aarch64-apple-ios" + || target == "aarch64-apple-ios-sim" + || target == "x86_64-apple-ios" + { + // Update the library’s install name to reflect that it has been renamed. + apple_darwin_update_library_name(builder, &dst, &format!("@rpath/{}", runtime.name)); + // Upon renaming the install name, the code signature of the file will invalidate, + // so we will sign it again. + apple_darwin_sign_file(builder, &dst); + } + + target_deps.push(dst); + } + + target_deps +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_self_contained_objects.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_self_contained_objects.rs new file mode 100644 index 00000000..8718bfd5 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_self_contained_objects.rs @@ -0,0 +1,76 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_self_contained_objects` from `compile.rs`. + +/// Copies third party objects needed by various targets for self-contained linkage. +fn copy_self_contained_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let libdir_self_contained = + builder.sysroot_target_libdir(*compiler, target).join("self-contained"); + t!(fs::create_dir_all(&libdir_self_contained)); + let mut target_deps = vec![]; + + // Copies the libc and CRT objects. + // + // rustc historically provides a more self-contained installation for musl targets + // not requiring the presence of a native musl toolchain. For example, it can fall back + // to using gcc from a glibc-targeting toolchain for linking. + // To do that we have to distribute musl startup objects as a part of Rust toolchain + // and link with them manually in the self-contained mode. + if target.contains("musl") && !target.contains("unikraft") { + let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { + panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) + }); + for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); + for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { + let src = crt_path.join(obj); + let target = libdir_self_contained.join(obj); + builder.copy_link(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + + if !target.starts_with("s390x") { + let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); + target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); + } + } else if target.contains("-wasi") { + let srcdir = builder.wasi_libdir(target).unwrap_or_else(|| { + panic!( + "Target {:?} does not have a \"wasi-root\" key in Config.toml \ + or `$WASI_SDK_PATH` set", + target.triple + ) + }); + for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + } else if target.is_windows_gnu() { + for obj in ["crt2.o", "dllcrt2.o"].iter() { + let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); + let target = libdir_self_contained.join(obj); + builder.copy_link(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + } + + target_deps +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_third_party_objects.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_third_party_objects.rs new file mode 100644 index 00000000..a4d5677b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/copy_third_party_objects.rs @@ -0,0 +1,32 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_third_party_objects` from `compile.rs`. + +/// Copies third party objects needed by various targets. +fn copy_third_party_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let mut target_deps = vec![]; + + if builder.config.needs_sanitizer_runtime_built(target) && compiler.stage != 0 { + // The sanitizers are only copied in stage1 or above, + // to avoid creating dependency on LLVM. + target_deps.extend( + copy_sanitizers(builder, compiler, target) + .into_iter() + .map(|d| (d, DependencyType::Target)), + ); + } + + if target == "x86_64-fortanix-unknown-sgx" + || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree + && (target.contains("linux") || target.contains("fuchsia")) + { + let libunwind_path = + copy_llvm_libunwind(builder, target, &builder.sysroot_target_libdir(*compiler, target)); + target_deps.push((libunwind_path, DependencyType::Target)); + } + + target_deps +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cp_rustc_component_to_ci_sysroot.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cp_rustc_component_to_ci_sysroot.rs new file mode 100644 index 00000000..07b97ab6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/cp_rustc_component_to_ci_sysroot.rs @@ -0,0 +1,16 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `cp_rustc_component_to_ci_sysroot` from `compile.rs`. + +fn cp_rustc_component_to_ci_sysroot(builder: &Builder<'_>, sysroot: &Path, contents: Vec) { + let ci_rustc_dir = builder.config.ci_rustc_dir(); + + for file in contents { + let src = ci_rustc_dir.join(&file); + let dst = sysroot.join(file); + if src.is_dir() { + t!(fs::create_dir_all(dst)); + } else { + builder.copy_link(&src, &dst); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/librustc_stamp.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/librustc_stamp.rs new file mode 100644 index 00000000..503e06bc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/librustc_stamp.rs @@ -0,0 +1,12 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `librustc_stamp` from `compile.rs`. + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +pub fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/libstd_stamp.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/libstd_stamp.rs new file mode 100644 index 00000000..a9bd241d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/libstd_stamp.rs @@ -0,0 +1,8 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `libstd_stamp` from `compile.rs`. + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/mod.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/mod.rs new file mode 100644 index 00000000..c1990c6a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/mod.rs @@ -0,0 +1,32 @@ +pub mod add_to_sysroot; +pub mod apple_darwin_sign_file; +pub mod apple_darwin_update_library_name; +pub mod assemble; +pub mod cargo_message; +pub mod cargo_target; +pub mod codegen_backend_stamp; +pub mod codegen_backend; +pub mod compiler_file; +pub mod compiler_rt_for_profiler; +pub mod copy_and_stamp; +pub mod copy_llvm_libunwind; +pub mod copy_sanitizers; +pub mod copy_self_contained_objects; +pub mod copy_third_party_objects; +pub mod cp_rustc_component_to_ci_sysroot; +pub mod librustc_stamp; +pub mod libstd_stamp; +pub mod run_cargo; +pub mod rustc_cargo_env; +pub mod rustc_cargo; +pub mod rustc_link; +pub mod rustc_llvm_env; +pub mod rustc; +pub mod startup_objects; +pub mod std_cargo; +pub mod std_crates_for_run_make; +pub mod std_link; +pub mod std; +pub mod stream_cargo; +pub mod strip_debug; +pub mod sysroot; diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/run_cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/run_cargo.rs new file mode 100644 index 00000000..dc452e26 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/run_cargo.rs @@ -0,0 +1,166 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `run_cargo` from `compile.rs`. + +pub fn run_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + stamp: &Path, + additional_target_deps: Vec<(PathBuf, DependencyType)>, + is_check: bool, + rlib_only_metadata: bool, +) -> Vec { + // `target_root_dir` looks like $dir/$target/release + let target_root_dir = stamp.parent().unwrap(); + // `target_deps_dir` looks like $dir/$target/release/deps + let target_deps_dir = target_root_dir.join("deps"); + // `host_root_dir` looks like $dir/release + let host_root_dir = target_root_dir + .parent() + .unwrap() // chop off `release` + .parent() + .unwrap() // chop off `$target` + .join(target_root_dir.file_name().unwrap()); + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let mut deps = Vec::new(); + let mut toplevel = Vec::new(); + let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { + let (filenames, crate_types) = match msg { + CargoMessage::CompilerArtifact { + filenames, + target: CargoTarget { crate_types }, + .. + } => (filenames, crate_types), + _ => return, + }; + for filename in filenames { + // Skip files like executables + let mut keep = false; + if filename.ends_with(".lib") + || filename.ends_with(".a") + || is_debug_info(&filename) + || is_dylib(Path::new(&*filename)) + { + // Always keep native libraries, rust dylibs and debuginfo + keep = true; + } + if is_check && filename.ends_with(".rmeta") { + // During check builds we need to keep crate metadata + keep = true; + } else if rlib_only_metadata { + if filename.contains("jemalloc_sys") + || filename.contains("rustc_smir") + || filename.contains("stable_mir") + { + // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, + // so we need to distribute them as rlib to be able to use them. + keep |= filename.ends_with(".rlib"); + } else { + // Distribute the rest of the rustc crates as rmeta files only to reduce + // the tarball sizes by about 50%. The object files are linked into + // librustc_driver.so, so it is still possible to link against them. + keep |= filename.ends_with(".rmeta"); + } + } + else { + // In all other cases keep all rlibs + keep |= filename.ends_with(".rlib"); + } + + if !keep { + continue; + } + + let filename = Path::new(&*filename); + + // If this was an output file in the "host dir" we don't actually + // worry about it, it's not relevant for us + if filename.starts_with(&host_root_dir) { + // Unless it's a proc macro used in the compiler + if crate_types.iter().any(|t| t == "proc-macro") { + deps.push((filename.to_path_buf(), DependencyType::Host)); + } + continue; + } + + // If this was output in the `deps` dir then this is a precise file + // name (hash included) so we start tracking it. + if filename.starts_with(&target_deps_dir) { + deps.push((filename.to_path_buf(), DependencyType::Target)); + continue; + } + + // Otherwise this was a "top level artifact" which right now doesn't + // have a hash in the name, but there's a version of this file in + // the `deps` folder which *does* have a hash in the name. That's + // the one we'll want to we'll probe for it later. + // + // We do not use `Path::file_stem` or `Path::extension` here, + // because some generated files may have multiple extensions e.g. + // `std-.dll.lib` on Windows. The aforementioned methods only + // split the file name by the last extension (`.lib`) while we need + // to split by all extensions (`.dll.lib`). + let expected_len = t!(filename.metadata()).len(); + let filename = filename.file_name().unwrap().to_str().unwrap(); + let mut parts = filename.splitn(2, '.'); + let file_stem = parts.next().unwrap().to_owned(); + let extension = parts.next().unwrap().to_owned(); + + toplevel.push((file_stem, extension, expected_len)); + } + }); + + if builder.config.dry_run { + return Vec::new(); + } + + // Ok now we need to actually find all the files listed in `toplevel`. We've + // got a list of prefix/extensions and we basically just need to find the + // most recent file in the `deps` folder corresponding to each one. + let contents = t!(target_deps_dir.read_dir()) + .map(|e| t!(e)) + .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) + .collect::>(); + for (prefix, extension, expected_len) in toplevel { + let candidates = contents.iter().filter(|&(_, filename, meta)| { + meta.len() == expected_len + && filename + .strip_prefix(&prefix[..]) + .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) + .unwrap_or(false) + }); + let max = candidates.max_by_key(|&(_, _, metadata)| { + metadata.modified().expect("mtime should be available on all relevant OSes") + }); + let path_to_add = match max { + Some(triple) => triple.0.to_str().unwrap(), + None => panic!("no output generated for {prefix:?} {extension:?}"), + }; + if is_dylib(Path::new(path_to_add)) { + let candidate = format!("{path_to_add}.lib"); + let candidate = PathBuf::from(candidate); + if candidate.exists() { + deps.push((candidate, DependencyType::Target)); + } + } + deps.push((path_to_add.into(), DependencyType::Target)); + } + + deps.extend(additional_target_deps); + deps.sort(); + let mut new_contents = Vec::new(); + for (dep, dependency_type) in deps.iter() { + new_contents.extend(match *dependency_type { + DependencyType::Host => b"h", + DependencyType::Target => b"t", + DependencyType::TargetSelfContained => b"s", + }); + new_contents.extend(dep.to_str().unwrap().as_bytes()); + new_contents.extend(b"\0"); + } + t!(fs::write(stamp, &new_contents)); + deps.into_iter().map(|(d, _)| d).collect() +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc.rs new file mode 100644 index 00000000..a837126b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc.rs @@ -0,0 +1,178 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Rustc` from `compile.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + /// The **previous** compiler used to compile this compiler. + pub compiler: Compiler, + /// Whether to build a subset of crates, rather than the whole compiler. + /// + /// This should only be requested by the user, not used within bootstrap itself. + /// Using it within bootstrap can lead to confusing situation where lints are replayed + /// in two different steps. + crates: Vec, +} + +impl Rustc { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { target, compiler, crates: Default::default() } + } +} + +impl Step for Rustc { + /// We return the stage of the "actual" compiler (not the uplifted one). + /// + /// By "actual" we refer to the uplifting logic where we may not compile the requested stage; + /// instead, we uplift it from the previous stages. Which can lead to bootstrap failures in + /// specific situations where we request stage X from other steps. However we may end up + /// uplifting it from stage Y, causing the other stage to fail when attempting to link with + /// stage X which was never actually built. + type Output = u32; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let mut crates = run.builder.in_tree_crates("rustc-main", None); + for (i, krate) in crates.iter().enumerate() { + // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. + // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. + if krate.name == "rustc-main" { + crates.swap_remove(i); + break; + } + } + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.cargo_crates_in_set(); + run.builder.ensure(Rustc { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) -> u32 { + let compiler = self.compiler; + let target = self.target; + + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.download_rustc() && compiler.stage != 0 { + let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + cp_rustc_component_to_ci_sysroot( + builder, + &sysroot, + builder.config.ci_rustc_dev_contents(), + ); + return compiler.stage; + } + + builder.ensure(Std::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info("WARNING: Using a potentially old librustc. This may not behave well."); + builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); + builder.ensure(RustcLink::from_rustc(self, compiler)); + + return compiler.stage; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Rustc::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting rustc (stage{} -> stage{})", + compiler_to_use.stage, + compiler.stage + 1 + ) + } else { + format!( + "Uplifting rustc (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, + compiler_to_use.host, + compiler.stage + 1, + target + ) + }; + builder.info(&msg); + builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); + return compiler_to_use.stage; + } + + // Ensure that build scripts and proc macros have a std / libproc_macro to link against. + builder.ensure(Std::new( + builder.compiler(self.compiler.stage, builder.config.build), + builder.config.build, + )); + + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Rustc, + SourceType::InTree, + target, + Kind::Build, + ); + + rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); + + // NB: all RUSTFLAGS should be added to `rustc_cargo()` so they will be + // consistently applied by check/doc/test modes too. + + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + if builder.build.config.enable_bolt_settings && compiler.stage == 1 { + // Relocations are required for BOLT to work. + cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); + } + + let _guard = builder.msg_sysroot_tool( + Kind::Build, + compiler.stage, + format_args!("compiler artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + let stamp = librustc_stamp(builder, compiler, target); + run_cargo( + builder, + cargo, + vec![], + &stamp, + vec![], + false, + true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. + ); + + // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain + // unexpected debuginfo from dependencies, for example from the C++ standard library used in + // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with + // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo + // away after the fact. + if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None + && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None + { + let target_root_dir = stamp.parent().unwrap(); + let rustc_driver = target_root_dir.join("librustc_driver.so"); + strip_debug(builder, target, &rustc_driver); + } + + builder.ensure(RustcLink::from_rustc( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + + compiler.stage + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo.rs new file mode 100644 index 00000000..07bd856a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo.rs @@ -0,0 +1,120 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `rustc_cargo` from `compile.rs`. + +pub fn rustc_cargo( + builder: &Builder<'_>, + cargo: &mut Cargo, + target: TargetSelection, + compiler: &Compiler, + crates: &[String], +) { + cargo + .arg("--features") + .arg(builder.rustc_features(builder.kind, target, crates)) + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc/Cargo.toml")); + + cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); + + // If the rustc output is piped to e.g. `head -n1` we want the process to be killed, rather than + // having an error bubble up and cause a panic. + // + // FIXME(jieyouxu): this flag is load-bearing for rustc to not ICE on broken pipes, because + // rustc internally sometimes uses std `println!` -- but std `println!` by default will panic on + // broken pipes, and uncaught panics will manifest as an ICE. The compiler *should* handle this + // properly, but this flag is set in the meantime to paper over the I/O errors. + // + // See for details. + // + // Also see the discussion for properly handling I/O errors related to broken pipes, i.e. safe + // variants of `println!` in + // . + cargo.rustflag("-Zon-broken-pipe=kill"); + + if builder.config.llvm_enzyme { + cargo.rustflag("-l").rustflag("Enzyme-19"); + } + + // Building with protected visibility reduces the number of dynamic relocations needed, giving + // us a faster startup time. However GNU ld < 2.40 will error if we try to link a shared object + // with direct references to protected symbols, so for now we only use protected symbols if + // linking with LLD is enabled. + if builder.build.config.lld_mode.is_used() && !compiler.host.is_msvc() { + cargo.rustflag("-Zdefault-visibility=protected"); + } + + // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary + // and may just be a time sink. + if compiler.stage != 0 { + match builder.config.rust_lto { + RustcLto::Thin | RustcLto::Fat => { + // Since using LTO for optimizing dylibs is currently experimental, + // we need to pass -Zdylib-lto. + cargo.rustflag("-Zdylib-lto"); + // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when + // compiling dylibs (and their dependencies), even when LTO is enabled for the + // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. + let lto_type = match builder.config.rust_lto { + RustcLto::Thin => "thin", + RustcLto::Fat => "fat", + _ => unreachable!(), + }; + cargo.rustflag(&format!("-Clto={lto_type}")); + cargo.rustflag("-Cembed-bitcode=yes"); + } + RustcLto::ThinLocal => { /* Do nothing, this is the default */ } + RustcLto::Off => { + cargo.rustflag("-Clto=off"); + } + } + } else if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + // With LLD, we can use ICF (identical code folding) to reduce the executable size + // of librustc_driver/rustc and to improve i-cache utilization. + // + // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) + // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: + // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 + // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 + if builder.config.lld_mode.is_used() && !compiler.host.is_msvc() { + cargo.rustflag("-Clink-args=-Wl,--icf=all"); + } + + if builder.config.rust_profile_use.is_some() && builder.config.rust_profile_generate.is_some() { + panic!("Cannot use and generate PGO profiles at the same time"); + } + let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-generate={path}")); + // Apparently necessary to avoid overflowing the counters during + // a Cargo build profile + cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); + true + } else { + false + } + } else if let Some(path) = &builder.config.rust_profile_use { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-use={path}")); + if builder.is_verbose() { + cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); + } + true + } else { + false + } + } else { + false + }; + if is_collecting { + // Ensure paths to Rust sources are relative, not absolute. + cargo.rustflag(&format!( + "-Cllvm-args=-static-func-strip-dirname-prefix={}", + builder.config.src.components().count() + )); + } + + rustc_cargo_env(builder, cargo, target, compiler.stage); +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo_env.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo_env.rs new file mode 100644 index 00000000..650bea25 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_cargo_env.rs @@ -0,0 +1,81 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `rustc_cargo_env` from `compile.rs`. + +pub fn rustc_cargo_env( + builder: &Builder<'_>, + cargo: &mut Cargo, + target: TargetSelection, + stage: u32, +) { + // Set some configuration variables picked up by build scripts and + // the compiler alike + cargo + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("CFG_VERSION", builder.rust_version()); + + // Some tools like Cargo detect their own git information in build scripts. When omit-git-hash + // is enabled in config.toml, we pass this environment variable to tell build scripts to avoid + // detecting git information on their own. + if builder.config.omit_git_hash { + cargo.env("CFG_OMIT_GIT_HASH", "1"); + } + + if let Some(backend) = builder.config.default_codegen_backend(target) { + cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); + } + + let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); + let target_config = builder.config.target_config.get(&target); + + cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); + + if let Some(ref ver_date) = builder.rust_info().commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info().sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + // Prefer the current target's own default_linker, else a globally + // specified one. + if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { + cargo.env("CFG_DEFAULT_LINKER", s); + } else if let Some(ref s) = builder.config.rustc_default_linker { + cargo.env("CFG_DEFAULT_LINKER", s); + } + + // Enable rustc's env var for `rust-lld` when requested. + if builder.config.lld_enabled + && (builder.config.channel == "dev" || builder.config.channel == "nightly") + { + cargo.env("CFG_USE_SELF_CONTAINED_LINKER", "1"); + } + + if builder.config.rust_verify_llvm_ir { + cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); + } + + if builder.config.llvm_enzyme { + cargo.rustflag("--cfg=llvm_enzyme"); + } + + // Note that this is disabled if LLVM itself is disabled or we're in a check + // build. If we are in a check build we still go ahead here presuming we've + // detected that LLVM is already built and good to go which helps prevent + // busting caches (e.g. like #71152). + if builder.config.llvm_enabled(target) { + let building_is_expensive = + crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target, false) + .should_build(); + // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler + let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; + let should_skip_build = building_is_expensive && can_skip_build; + if !should_skip_build { + rustc_llvm_env(builder, cargo, target) + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_link.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_link.rs new file mode 100644 index 00000000..19df2166 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_link.rs @@ -0,0 +1,50 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `RustcLink` from `compile.rs`. + +/// `RustcLink` copies all of the rlibs from the rustc build into the previous stage's sysroot. +/// This is necessary for tools using `rustc_private`, where the previous compiler will build +/// a tool against the next compiler. +/// To build a tool against a compiler, the rlibs of that compiler that it links against +/// must be in the sysroot of the compiler that's doing the compiling. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct RustcLink { + /// The compiler whose rlibs we are copying around. + pub compiler: Compiler, + /// This is the compiler into whose sysroot we want to copy the rlibs into. + pub previous_stage_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Vec, +} + +impl RustcLink { + fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + previous_stage_compiler: rustc.compiler, + target: rustc.target, + crates: rustc.crates, + } + } +} + +impl Step for RustcLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Same as `std_link`, only for librustc + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let previous_stage_compiler = self.previous_stage_compiler; + let target = self.target; + add_to_sysroot( + builder, + &builder.sysroot_target_libdir(previous_stage_compiler, target), + &builder.sysroot_target_libdir(previous_stage_compiler, compiler.host), + &librustc_stamp(builder, compiler, target), + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_llvm_env.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_llvm_env.rs new file mode 100644 index 00000000..15c0d028 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/rustc_llvm_env.rs @@ -0,0 +1,70 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `rustc_llvm_env` from `compile.rs`. + +/// Pass down configuration from the LLVM build into the build of +/// rustc_llvm and rustc_codegen_llvm. +fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + if builder.is_rust_llvm(target) { + cargo.env("LLVM_RUSTLLVM", "1"); + } + let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + cargo.env("LLVM_CONFIG", &llvm_config); + + // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script + // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by + // whitespace. + // + // For example: + // - on windows, when `clang-cl` is used with instrumentation, we need to manually add + // clang's runtime library resource directory so that the profiler runtime library can be + // found. This is to avoid the linker errors about undefined references to + // `__llvm_profile_instrument_memop` when linking `rustc_driver`. + let mut llvm_linker_flags = String::new(); + if builder.config.llvm_profile_generate && target.is_msvc() { + if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { + // Add clang's runtime library directory to the search path + let clang_rt_dir = get_clang_cl_resource_dir(builder, clang_cl_path); + llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); + } + } + + // The config can also specify its own llvm linker flags. + if let Some(ref s) = builder.config.llvm_ldflags { + if !llvm_linker_flags.is_empty() { + llvm_linker_flags.push(' '); + } + llvm_linker_flags.push_str(s); + } + + // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + if !llvm_linker_flags.is_empty() { + cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + } + + // Building with a static libstdc++ is only supported on linux right now, + // not for MSVC or macOS + if builder.config.llvm_static_stdcpp + && !target.contains("freebsd") + && !target.is_msvc() + && !target.contains("apple") + && !target.contains("solaris") + { + let file = compiler_file( + builder, + &builder.cxx(target).unwrap(), + target, + CLang::Cxx, + "libstdc++.a", + ); + cargo.env("LLVM_STATIC_STDCPP", file); + } + if builder.llvm_link_shared() { + cargo.env("LLVM_LINK_SHARED", "1"); + } + if builder.config.llvm_use_libcxx { + cargo.env("LLVM_USE_LIBCXX", "1"); + } + if builder.config.llvm_assertions { + cargo.env("LLVM_ASSERTIONS", "1"); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/startup_objects.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/startup_objects.rs new file mode 100644 index 00000000..2f5f737d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/startup_objects.rs @@ -0,0 +1,70 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `StartupObjects` from `compile.rs`. + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct StartupObjects { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for StartupObjects { + type Output = Vec<(PathBuf, DependencyType)>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("library/rtstartup") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(StartupObjects { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + }); + } + + /// Builds and prepare startup objects like rsbegin.o and rsend.o + /// + /// These are primarily used on Windows right now for linking executables/dlls. + /// They don't require any library support as they're just plain old object + /// files, so we just use the nightly snapshot compiler to always build them (as + /// no other compilers are guaranteed to be available). + fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { + let for_compiler = self.compiler; + let target = self.target; + if !target.is_windows_gnu() { + return vec![]; + } + + let mut target_deps = vec![]; + + let src_dir = &builder.src.join("library").join("rtstartup"); + let dst_dir = &builder.native_dir(target).join("rtstartup"); + let sysroot_dir = &builder.sysroot_target_libdir(for_compiler, target); + t!(fs::create_dir_all(dst_dir)); + + for file in &["rsbegin", "rsend"] { + let src_file = &src_dir.join(file.to_string() + ".rs"); + let dst_file = &dst_dir.join(file.to_string() + ".o"); + if !up_to_date(src_file, dst_file) { + let mut cmd = command(&builder.initial_rustc); + cmd.env("RUSTC_BOOTSTRAP", "1"); + if !builder.local_rebuild { + // a local_rebuild compiler already has stage1 features + cmd.arg("--cfg").arg("bootstrap"); + } + cmd.arg("--target") + .arg(target.rustc_target_arg()) + .arg("--emit=obj") + .arg("-o") + .arg(dst_file) + .arg(src_file) + .run(builder); + } + + let target = sysroot_dir.join((*file).to_string() + ".o"); + builder.copy_link(dst_file, &target); + target_deps.push((target, DependencyType::Target)); + } + + target_deps + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std.rs new file mode 100644 index 00000000..11bfdfbf --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std.rs @@ -0,0 +1,260 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Std` from `compile.rs`. + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Std { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build only a subset of crates in the standard library. + /// + /// This shouldn't be used from other steps; see the comment on [`Rustc`]. + crates: Vec, + /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, + /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overridden by `builder.ensure` from other steps. + force_recompile: bool, + extra_rust_args: &'static [&'static str], + is_for_mir_opt_tests: bool, +} + +impl Std { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + is_for_mir_opt_tests: false, + } + } + + pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: true, + extra_rust_args: &[], + is_for_mir_opt_tests: false, + } + } + + pub fn new_for_mir_opt_tests(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + is_for_mir_opt_tests: true, + } + } + + pub fn new_with_extra_rust_args( + compiler: Compiler, + target: TargetSelection, + extra_rust_args: &'static [&'static str], + ) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args, + is_for_mir_opt_tests: false, + } + } + + fn copy_extra_objects( + &self, + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, + ) -> Vec<(PathBuf, DependencyType)> { + let mut deps = Vec::new(); + if !self.is_for_mir_opt_tests { + deps.extend(copy_third_party_objects(builder, compiler, target)); + deps.extend(copy_self_contained_objects(builder, compiler, target)); + } + deps + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot").path("library") + } + + fn make_run(run: RunConfig<'_>) { + let crates = std_crates_for_run_make(&run); + let builder = run.builder; + + // Force compilation of the standard library from source if the `library` is modified. This allows + // library team to compile the standard library without needing to compile the compiler with + // the `rust.download-rustc=true` option. + let force_recompile = builder.rust_info().is_managed_git_subrepository() + && builder.download_rustc() + && builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none(); + + run.builder.ensure(Std { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + force_recompile, + extra_rust_args: &[], + is_for_mir_opt_tests: false, + }); + } + + /// Builds the standard library. + /// + /// This will build the standard library for a particular stage of the build + /// using the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + + // When using `download-rustc`, we already have artifacts for the host available. Don't + // recompile them. + if builder.download_rustc() && target == builder.build.build + // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so + // its artifacts can't be reused. + && compiler.stage != 0 + && !self.force_recompile + { + let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + cp_rustc_component_to_ci_sysroot( + builder, + &sysroot, + builder.config.ci_rust_std_contents(), + ); + return; + } + + if builder.config.keep_stage.contains(&compiler.stage) + || builder.config.keep_stage_std.contains(&compiler.stage) + { + builder.info("WARNING: Using a potentially old libstd. This may not behave well."); + + builder.ensure(StartupObjects { compiler, target }); + + self.copy_extra_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler)); + return; + } + + builder.require_submodule("library/stdarch", None); + + let mut target_deps = builder.ensure(StartupObjects { compiler, target }); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Std::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting library (stage{} -> stage{})", + compiler_to_use.stage, compiler.stage + ) + } else { + format!( + "Uplifting library (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, compiler_to_use.host, compiler.stage, target + ) + }; + builder.info(&msg); + + // Even if we're not building std this stage, the new sysroot must + // still contain the third party objects needed by various targets. + self.copy_extra_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler_to_use)); + return; + } + + target_deps.extend(self.copy_extra_objects(builder, &compiler, target)); + + // The LLD wrappers and `rust-lld` are self-contained linking components that can be + // necessary to link the stdlib on some targets. We'll also need to copy these binaries to + // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. + if compiler.stage == 0 && compiler.host == builder.config.build { + // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. + let src_sysroot_bin = builder + .rustc_snapshot_sysroot() + .join("lib") + .join("rustlib") + .join(compiler.host) + .join("bin"); + if src_sysroot_bin.exists() { + let target_sysroot_bin = builder.sysroot_target_bindir(compiler, target); + t!(fs::create_dir_all(&target_sysroot_bin)); + builder.cp_link_r(&src_sysroot_bin, &target_sysroot_bin); + } + } + + // We build a sysroot for mir-opt tests using the same trick that Miri does: A check build + // with -Zalways-encode-mir. This frees us from the need to have a target linker, and the + // fact that this is a check build integrates nicely with run_cargo. + let mut cargo = if self.is_for_mir_opt_tests { + let mut cargo = builder::Cargo::new_for_mir_opt_tests( + builder, + compiler, + Mode::Std, + SourceType::InTree, + target, + Kind::Check, + ); + cargo.rustflag("-Zalways-encode-mir"); + cargo.arg("--manifest-path").arg(builder.src.join("library/sysroot/Cargo.toml")); + cargo + } else { + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Std, + SourceType::InTree, + target, + Kind::Build, + ); + std_cargo(builder, target, compiler.stage, &mut cargo); + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + cargo + }; + + // See src/bootstrap/synthetic_targets.rs + if target.is_synthetic() { + cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); + } + for rustflag in self.extra_rust_args.iter() { + cargo.rustflag(rustflag); + } + + let _guard = builder.msg( + Kind::Build, + compiler.stage, + format_args!("library artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + run_cargo( + builder, + cargo, + vec![], + &libstd_stamp(builder, compiler, target), + target_deps, + self.is_for_mir_opt_tests, // is_check + false, + ); + + builder.ensure(StdLink::from_std( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_cargo.rs new file mode 100644 index 00000000..ce015cbd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_cargo.rs @@ -0,0 +1,137 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `std_cargo` from `compile.rs`. + +/// Configure cargo to compile the standard library, adding appropriate env vars +/// and such. +pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { + if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { + cargo.env("MACOSX_DEPLOYMENT_TARGET", target); + } + + // Paths needed by `library/profiler_builtins/build.rs`. + if let Some(path) = builder.config.profiler_path(target) { + cargo.env("LLVM_PROFILER_RT_LIB", path); + } else if builder.config.profiler_enabled(target) { + let compiler_rt = compiler_rt_for_profiler(builder); + // Currently this is separate from the env var used by `compiler_builtins` + // (below) so that adding support for CI LLVM here doesn't risk breaking + // the compiler builtins. But they could be unified if desired. + cargo.env("RUST_COMPILER_RT_FOR_PROFILER", compiler_rt); + } + + // Determine if we're going to compile in optimized C intrinsics to + // the `compiler-builtins` crate. These intrinsics live in LLVM's + // `compiler-rt` repository. + // + // Note that this shouldn't affect the correctness of `compiler-builtins`, + // but only its speed. Some intrinsics in C haven't been translated to Rust + // yet but that's pretty rare. Other intrinsics have optimized + // implementations in C which have only had slower versions ported to Rust, + // so we favor the C version where we can, but it's not critical. + // + // If `compiler-rt` is available ensure that the `c` feature of the + // `compiler-builtins` crate is enabled and it's configured to learn where + // `compiler-rt` is located. + let compiler_builtins_c_feature = if builder.config.optimized_compiler_builtins { + // NOTE: this interacts strangely with `llvm-has-rust-patches`. In that case, we enforce `submodules = false`, so this is a no-op. + // But, the user could still decide to manually use an in-tree submodule. + // + // NOTE: if we're using system llvm, we'll end up building a version of `compiler-rt` that doesn't match the LLVM we're linking to. + // That's probably ok? At least, the difference wasn't enforced before. There's a comment in + // the compiler_builtins build script that makes me nervous, though: + // https://github.com/rust-lang/compiler-builtins/blob/31ee4544dbe47903ce771270d6e3bea8654e9e50/build.rs#L575-L579 + builder.require_submodule( + "src/llvm-project", + Some( + include_str!("std_cargo_warning_1.txt"), ), + ); + let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); + assert!(compiler_builtins_root.exists()); + // The path to `compiler-rt` is also used by `profiler_builtins` (above), + // so if you're changing something here please also change that as appropriate. + cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); + " compiler-builtins-c" + } else { + "" + }; + + // `libtest` uses this to know whether or not to support + // `-Zunstable-options`. + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + let mut features = String::new(); + + if builder.no_std(target) == Some(true) { + features += " compiler-builtins-mem"; + if !target.starts_with("sbf") && !target.starts_with("bpf") { + features.push_str(compiler_builtins_c_feature); + } + + // for no-std targets we only compile a few no_std crates + cargo + .args(["-p", "alloc"]) + .arg("--manifest-path") + .arg(builder.src.join("library/alloc/Cargo.toml")) + .arg("--features") + .arg(features); + } else { + features += &builder.std_features(target); + features.push_str(compiler_builtins_c_feature); + + cargo + .arg("--features") + .arg(features) + .arg("--manifest-path") + .arg(builder.src.join("library/sysroot/Cargo.toml")); + + // Help the libc crate compile by assisting it in finding various + // sysroot native libraries. + if target.contains("musl") { + if let Some(p) = builder.musl_libdir(target) { + let root = format!("native={}", p.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + + if target.contains("-wasi") { + if let Some(dir) = builder.wasi_libdir(target) { + let root = format!("native={}", dir.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + } + + // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that + // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be + // built with bitcode so that the produced rlibs can be used for both LTO + // builds (which use bitcode) and non-LTO builds (which use object code). + // So we override the override here! + // + // But we don't bother for the stage 0 compiler because it's never used + // with LTO. + if stage >= 1 { + cargo.rustflag("-Cembed-bitcode=yes"); + } + if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + // By default, rustc does not include unwind tables unless they are required + // for a particular target. They are not required by RISC-V targets, but + // compiling the standard library with them means that users can get + // backtraces without having to recompile the standard library themselves. + // + // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 + if target.contains("riscv") { + cargo.rustflag("-Cforce-unwind-tables=yes"); + } + + // Enable frame pointers by default for the library. Note that they are still controlled by a + // separate setting for the compiler. + cargo.rustflag("-Cforce-frame-pointers=yes"); + +// let html_root = +// String::new(); +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_crates_for_run_make.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_crates_for_run_make.rs new file mode 100644 index 00000000..d84582b9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_crates_for_run_make.rs @@ -0,0 +1,24 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `std_crates_for_run_make` from `compile.rs`. + +/// Resolves standard library crates for `Std::run_make` for any build kind (like check, build, clippy, etc.). +pub fn std_crates_for_run_make(run: &RunConfig<'_>) -> Vec { + // FIXME: Extend builder tests to cover the `crates` field of `Std` instances. + if cfg!(feature = "bootstrap-self-test") { + return vec![]; + } + + let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); + let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false); + + // For no_std targets, do not add any additional crates to the compilation other than what `compile::std_cargo` already adds for no_std targets. + if target_is_no_std { + vec![] + } + // If the paths include "library", build the entire standard library. + else if has_alias { + run.make_run_crates(builder::Alias::Library) + } else { + run.cargo_crates_in_set() + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_link.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_link.rs new file mode 100644 index 00000000..58e6f529 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/std_link.rs @@ -0,0 +1,112 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `StdLink` from `compile.rs`. + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct StdLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Vec, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl StdLink { + fn from_std(std: Std, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: std.compiler, + target: std.target, + crates: std.crates, + force_recompile: std.force_recompile, + } + } +} + +impl Step for StdLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Link all libstd rlibs/dylibs into the sysroot location. + /// + /// Links those artifacts generated by `compiler` to the `stage` compiler's + /// sysroot for the specified `host` and `target`. + /// + /// Note that this assumes that `compiler` has already generated the libstd + /// libraries for `target`, and this method will find them in the relevant + /// output directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + + // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. + let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { + // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { + compiler: self.compiler, + force_recompile: self.force_recompile, + }); + let libdir = sysroot.join(lib).join("rustlib").join(target).join("lib"); + let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host).join("lib"); + (libdir, hostdir) + } else { + let libdir = builder.sysroot_target_libdir(target_compiler, target); + let hostdir = builder.sysroot_target_libdir(target_compiler, compiler.host); + (libdir, hostdir) + }; + + add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + + // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` + // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, + // and is not set to a custom path. + if compiler.stage == 0 + && builder + .build + .config + .initial_rustc + .starts_with(builder.out.join(compiler.host).join("stage0/bin")) + { + // Copy bin files from stage0/bin to stage0-sysroot/bin + let sysroot = builder.out.join(compiler.host).join("stage0-sysroot"); + + let host = compiler.host; + let stage0_bin_dir = builder.out.join(host).join("stage0/bin"); + let sysroot_bin_dir = sysroot.join("bin"); + t!(fs::create_dir_all(&sysroot_bin_dir)); + builder.cp_link_r(&stage0_bin_dir, &sysroot_bin_dir); + + // Copy all files from stage0/lib to stage0-sysroot/lib + let stage0_lib_dir = builder.out.join(host).join("stage0/lib"); + if let Ok(files) = fs::read_dir(stage0_lib_dir) { + for file in files { + let file = t!(file); + let path = file.path(); + if path.is_file() { + builder + .copy_link(&path, &sysroot.join("lib").join(path.file_name().unwrap())); + } + } + } + + // Copy codegen-backends from stage0 + let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); + t!(fs::create_dir_all(&sysroot_codegen_backends)); + let stage0_codegen_backends = builder + .out + .join(host) + .join("stage0/lib/rustlib") + .join(host) + .join("codegen-backends"); + if stage0_codegen_backends.exists() { + builder.cp_link_r(&stage0_codegen_backends, &sysroot_codegen_backends); + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/stream_cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/stream_cargo.rs new file mode 100644 index 00000000..6e0439de --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/stream_cargo.rs @@ -0,0 +1,66 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `stream_cargo` from `compile.rs`. + +pub fn stream_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + cb: &mut dyn FnMut(CargoMessage<'_>), +) -> bool { + let mut cmd = cargo.into_cmd(); + let cargo = cmd.as_command_mut(); + // Instruct Cargo to give us json messages on stdout, critically leaving + // stderr as piped so we can get those pretty colors. + let mut message_format = if builder.config.json_output { + String::from("json") + } else { + String::from("json-render-diagnostics") + }; + if let Some(s) = &builder.config.rustc_error_format { + message_format.push_str(",json-diagnostic-"); + message_format.push_str(s); + } + cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); + + for arg in tail_args { + cargo.arg(arg); + } + + builder.verbose(|| println!("running: {cargo:?}")); + + if builder.config.dry_run { + return true; + } + let mut child = match cargo.spawn() { + Ok(child) => child, + Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), + }; + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let stdout = BufReader::new(child.stdout.take().unwrap()); + for line in stdout.lines() { + let line = t!(line); + match serde_json::from_str::>(&line) { + Ok(msg) => { + if builder.config.json_output { + // Forward JSON to stdout. + println!("{line}"); + } + cb(msg) + } + // If this was informational, just print it out and continue + Err(_) => println!("{line}"), + } + } + + // Make sure Cargo actually succeeded after we read all of its stdout. + let status = t!(child.wait()); + if builder.is_verbose() && !status.success() { + eprintln!( + "command did not execute successfully: {cargo:?}\nexpected success, got: {status}" + ); + } + status.success() +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/strip_debug.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/strip_debug.rs new file mode 100644 index 00000000..b96ad364 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/strip_debug.rs @@ -0,0 +1,30 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `strip_debug` from `compile.rs`. + +pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { + // FIXME: to make things simpler for now, limit this to the host and target where we know + // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not + // cross-compiling. Expand this to other appropriate targets in the future. + if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { + return; + } + + let previous_mtime = t!(t!(path.metadata()).modified()); + command("strip").arg("--strip-debug").arg(path).run_capture(builder); + + let file = t!(fs::File::open(path)); + + // After running `strip`, we have to set the file modification time to what it was before, + // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time + // bootstrap is invoked. + // + // An example of this is if we run this on librustc_driver.so. In the first invocation: + // - Cargo will build librustc_driver.so (mtime of 1) + // - Cargo will build rustc-main (mtime of 2) + // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). + // + // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so + // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause + // everything else (standard library, future stages...) to be rebuilt. + t!(file.set_modified(previous_mtime)); +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/compile_modules/sysroot.rs b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/sysroot.rs new file mode 100644 index 00000000..e1eb919c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/compile_modules/sysroot.rs @@ -0,0 +1,167 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Sysroot` from `compile.rs`. + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Sysroot { + pub compiler: Compiler, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl Sysroot { + pub(crate) fn new(compiler: Compiler) -> Self { + Sysroot { compiler, force_recompile: false } + } +} + +impl Step for Sysroot { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Returns the sysroot that `compiler` is supposed to use. + /// For the stage0 compiler, this is stage0-sysroot (because of the initial std build). + /// For all other stages, it's the same stage directory that the compiler lives in. + fn run(self, builder: &Builder<'_>) -> PathBuf { + let compiler = self.compiler; + let host_dir = builder.out.join(compiler.host); + + let sysroot_dir = |stage| { + if stage == 0 { + host_dir.join("stage0-sysroot") + } else if self.force_recompile && stage == compiler.stage { + host_dir.join(format!("stage{stage}-test-sysroot")) + } else if builder.download_rustc() && compiler.stage != builder.top_stage { + host_dir.join("ci-rustc-sysroot") + } else { + host_dir.join(format!("stage{}", stage)) + } + }; + let sysroot = sysroot_dir(compiler.stage); + + builder + .verbose(|| println!("Removing sysroot {} to avoid caching bugs", sysroot.display())); + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + + // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 + // compiler relies on more recent version of LLVM than the beta compiler, it may not + // be able to locate the correct LLVM in the sysroot. This situation typically occurs + // when we upgrade LLVM version while the beta compiler continues to use an older version. + // + // Make sure to add the correct version of LLVM into the stage0 sysroot. + if compiler.stage == 0 { + dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); + } + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() && compiler.stage != 0 { + assert_eq!( + builder.config.build, compiler.host, + "Cross-compiling is not yet supported with `download-rustc`", + ); + + // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. + for stage in 0..=2 { + if stage != compiler.stage { + let dir = sysroot_dir(stage); + if !dir.ends_with("ci-rustc-sysroot") { + let _ = fs::remove_dir_all(dir); + } + } + } + + // Copy the compiler into the correct sysroot. + // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. + // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. + // There are a few quirks of bootstrap that interact to make this reliable: + // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This + // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to + // fail because of duplicate metadata. + // 2. The sysroot is deleted and recreated between each invocation, so running `x test + // ui-fulldeps && x test ui` can't cause failures. + let mut filtered_files = Vec::new(); + let mut add_filtered_files = |suffix, contents| { + for path in contents { + let path = Path::new(&path); + if path.parent().map_or(false, |parent| parent.ends_with(suffix)) { + filtered_files.push(path.file_name().unwrap().to_owned()); + } + } + }; + let suffix = format!("lib/rustlib/{}/lib", compiler.host); + add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); + // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the + // newly compiled std, not the downloaded std. + add_filtered_files("lib", builder.config.ci_rust_std_contents()); + + let filtered_extensions = [ + OsStr::new("rmeta"), + OsStr::new("rlib"), + // FIXME: this is wrong when compiler.host != build, but we don't support that today + OsStr::new(std::env::consts::DLL_EXTENSION), + ]; + let ci_rustc_dir = builder.config.ci_rustc_dir(); + builder.cp_link_filtered(&ci_rustc_dir, &sysroot, &|path| { + if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { + return true; + } + if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { + return true; + } + if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { + builder.verbose_than(1, || println!("ignoring {}", path.display())); + false + } else { + true + } + }); + } + + // Symlink the source root into the same location inside the sysroot, + // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), + // so that any tools relying on `rust-src` also work for local builds, + // and also for translating the virtual `/rustc/$hash` back to the real + // directory (for running tests with `rust.remap-debuginfo = true`). + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { + eprintln!( + "ERROR: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, + ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "ERROR: some `tests/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + build_helper::exit!(1); + } + + // rustc-src component is already part of CI rustc's sysroot + if !builder.download_rustc() { + let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); + let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); + if let Err(e) = + symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) + { + eprintln!( + "ERROR: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_rustcsrc_rust.display(), + builder.src.display(), + e, + ); + build_helper::exit!(1); + } + } + + sysroot + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist.rs b/standalonex/src/bootstrap/src/core/build_steps/dist.rs index cb352e21..c4d9f04e 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/dist.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/dist.rs @@ -173,7 +173,7 @@ fn make_win_dist( target: TargetSelection, builder: &Builder<'_>, ) { - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -912,7 +912,7 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - if !builder.config.dry_run() { + if !builder.config.dry_run { builder.require_submodule("src/llvm-project", None); } @@ -1333,7 +1333,7 @@ impl Step for CodegenBackend { } fn run(self, builder: &Builder<'_>) -> Option { - if builder.config.dry_run() { + if builder.config.dry_run { return None; } @@ -1511,7 +1511,7 @@ impl Step for Extended { let etc = builder.src.join("src/etc/installer"); // Avoid producing tarballs during a dry run. - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -1929,7 +1929,7 @@ impl Step for Extended { let _time = timeit(builder); cmd.run(builder); - if !builder.config.dry_run() { + if !builder.config.dry_run { t!(move_file(exe.join(&filename), distdir(builder).join(&filename))); } } @@ -1965,7 +1965,7 @@ fn install_llvm_file( destination: &Path, install_symlink: bool, ) { - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -2039,7 +2039,7 @@ fn maybe_install_llvm( if llvm_dylib_path.exists() { builder.install(&llvm_dylib_path, dst_libdir, 0o644); } - !builder.config.dry_run() + !builder.config.dry_run } else if let llvm::LlvmBuildStatus::AlreadyBuilt(llvm::LlvmResult { llvm_config, .. }) = llvm::prebuilt_llvm_config(builder, target, true) { @@ -2058,7 +2058,7 @@ fn maybe_install_llvm( }; install_llvm_file(builder, &file, dst_libdir, install_symlink); } - !builder.config.dry_run() + !builder.config.dry_run } else { false } diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/analysis.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/analysis.rs new file mode 100644 index 00000000..13c61557 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/analysis.rs @@ -0,0 +1,60 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Analysis` from `dist.rs`. + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct Analysis { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Analysis { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "analysis"); + run.alias("rust-analysis").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Analysis { + // Find the actual compiler (handling the full bootstrap option) which + // produced the save-analysis data because that data isn't copied + // through the sysroot uplifting. + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + /// Creates a tarball of (degenerate) save-analysis metadata, if available. + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if compiler.host != builder.config.build { + return None; + } + + let src = builder + .stage_out(compiler, Mode::Std) + .join(target) + .join(builder.cargo_dir()) + .join("deps") + .join("save-analysis"); + + // Write a file indicating that this component has been removed. + t!(std::fs::create_dir_all(&src)); + let mut removed = src.clone(); + removed.push("removed.json"); + let mut f = t!(std::fs::File::create(removed)); + t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); + + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/cargo.rs new file mode 100644 index 00000000..775a0703 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/cargo.rs @@ -0,0 +1,51 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Cargo` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "cargo"); + run.alias("cargo").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let cargo = builder.ensure(tool::Cargo { compiler, target }); + let src = builder.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/clippy.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/clippy.rs new file mode 100644 index 00000000..18865a3b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/clippy.rs @@ -0,0 +1,50 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Clippy` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Clippy { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Clippy { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "clippy"); + run.alias("clippy").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + // Prepare the image directory + // We expect clippy to build, because we've exited this step above if tool + // state for clippy isn't testing. + let clippy = builder.ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }); + let cargoclippy = + builder.ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }); + + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/codegen_backend.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/codegen_backend.rs new file mode 100644 index 00000000..ef7443f7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/codegen_backend.rs @@ -0,0 +1,90 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `CodegenBackend` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct CodegenBackend { + pub compiler: Compiler, + pub backend: String, +} + +impl Step for CodegenBackend { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc_codegen_cranelift") + } + + fn make_run(run: RunConfig<'_>) { + for backend in run.builder.config.codegen_backends(run.target) { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + backend: backend.clone(), + }); + } + } + + fn run(self, builder: &Builder<'_>) -> Option { + if builder.config.dry_run { + return None; + } + + // This prevents rustc_codegen_cranelift from being built for "dist" + // or "install" on the stable/beta channels. It is not yet stable and + // should not be included. + if !builder.build.unstable_features() { + return None; + } + + if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend.to_string()) + { + return None; + } + + if self.backend == "cranelift" && !target_supports_cranelift_backend(self.compiler.host) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return None; + } + + let compiler = self.compiler; + let backend = self.backend; + + let mut tarball = + Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); + if backend == "cranelift" { + tarball.set_overlay(OverlayKind::RustcCodegenCranelift); + } else { + panic!("Unknown backend rustc_codegen_{}", backend); + } + tarball.is_preview(true); + tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); + + let src = builder.sysroot(compiler); + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = PathBuf::from("lib").join(backends_rel); + + let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; + for backend in fs::read_dir(&backends_src).unwrap() { + let file_name = backend.unwrap().file_name(); + if file_name.to_str().unwrap().contains(&backend_name) { + tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; + } + } + assert!(found_backend); + + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_src_dirs.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_src_dirs.rs new file mode 100644 index 00000000..dbd0786e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_src_dirs.rs @@ -0,0 +1,96 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_src_dirs` from `dist.rs`. + +/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to +/// `dst_dir`. +fn copy_src_dirs( + builder: &Builder<'_>, + base: &Path, + src_dirs: &[&str], + exclude_dirs: &[&str], + dst_dir: &Path, +) { + fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { + let spath = match path.to_str() { + Some(path) => path, + None => return false, + }; + if spath.ends_with('~') || spath.ends_with(".pyc") { + return false; + } + + const LLVM_PROJECTS: &[&str] = &[ + "llvm-project/clang", + "llvm-project\clang", + "llvm-project/libunwind", + "llvm-project\libunwind", + "llvm-project/lld", + "llvm-project\lld", + "llvm-project/lldb", + "llvm-project\lldb", + "llvm-project/llvm", + "llvm-project\llvm", + "llvm-project/compiler-rt", + "llvm-project\compiler-rt", + "llvm-project/cmake", + "llvm-project\cmake", + "llvm-project/runtimes", + "llvm-project\runtimes", + ]; + if spath.contains("llvm-project") + && !spath.ends_with("llvm-project") + && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) { + return false; + } + + const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\llvm\test"]; + if LLVM_TEST.iter().any(|path| spath.contains(path)) + && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) { + return false; + } + + // Cargo tests use some files like `.gitignore` that we would otherwise exclude. + const CARGO_TESTS: &[&str] = &["tools/cargo/tests", "tools\cargo\tests"]; + if CARGO_TESTS.iter().any(|path| spath.contains(path)) { + return true; + } + + let full_path = Path::new(dir).join(path); + if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { + return false; + } + + let excludes = [ + "CVS", + "RCS", + "SCCS", + ".git", + ".gitignore", + ".gitmodules", + ".gitattributes", + ".cvsignore", + ".svn", + ".arch-ids", + "{arch}", + "=RELEASE-ID", + "=meta-update", + "=update", + ".bzr", + ".bzrignore", + ".bzrtags", + ".hg", + ".hgignore", + ".hgrags", + "_darcs", + ]; + !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) + } + + // Copy the directories using our filter + for item in src_dirs { + let dst = &dst_dir.join(item); + t!(fs::create_dir_all(dst)); + builder + .cp_link_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_target_libs.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_target_libs.rs new file mode 100644 index 00000000..e0f35981 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/copy_target_libs.rs @@ -0,0 +1,17 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `copy_target_libs` from `dist.rs`. + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target).join("lib"); + let self_contained_dst = dst.join("self-contained"); + t!(fs::create_dir_all(&dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + if dependency_type == DependencyType::TargetSelfContained { + builder.copy_link(&path, &self_contained_dst.join(path.file_name().unwrap())); + } else if dependency_type == DependencyType::Target || builder.config.build == target { + builder.copy_link(&path, &dst.join(path.file_name().unwrap())); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/debugger_scripts.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/debugger_scripts.rs new file mode 100644 index 00000000..c74ec35a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/debugger_scripts.rs @@ -0,0 +1,57 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `DebuggerScripts` from `dist.rs`. + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct DebuggerScripts { + pub sysroot: PathBuf, + pub host: TargetSelection, +} + +impl Step for DebuggerScripts { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Copies debugger scripts for `target` into the `sysroot` specified. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let sysroot = self.sysroot; + let dst = sysroot.join("lib/rustlib/etc"); + t!(fs::create_dir_all(&dst)); + let cp_debugger_script = |file: &str| { + builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); + }; + if host.contains("windows-msvc") { + // windbg debugger scripts + builder.install( + &builder.src.join("src/etc/rust-windbg.cmd"), + &sysroot.join("bin"), + 0o755, + ); + + cp_debugger_script("natvis/intrinsic.natvis"); + cp_debugger_script("natvis/liballoc.natvis"); + cp_debugger_script("natvis/libcore.natvis"); + cp_debugger_script("natvis/libstd.natvis"); + } else { + cp_debugger_script("rust_types.py"); + + // gdb debugger scripts + builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); + builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("gdb_load_rust_pretty_printers.py"); + cp_debugger_script("gdb_lookup.py"); + cp_debugger_script("gdb_providers.py"); + + // lldb debugger scripts + builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("lldb_lookup.py"); + cp_debugger_script("lldb_providers.py"); + cp_debugger_script("lldb_commands") + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/distdir.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/distdir.rs new file mode 100644 index 00000000..7f9b8b60 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/distdir.rs @@ -0,0 +1,6 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `distdir` from `dist.rs`. + +pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("dist") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/docs.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/docs.rs new file mode 100644 index 00000000..2a6fe11a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/docs.rs @@ -0,0 +1,35 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Docs` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Docs { + pub host: TargetSelection, +} + +impl Step for Docs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Docs { host: run.target }); + } + + /// Builds the `rust-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let dest = "share/doc/rust/html"; + + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_bulk_dir(builder.doc_out(host), dest); + tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/find_files.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/find_files.rs new file mode 100644 index 00000000..b75a6a54 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/find_files.rs @@ -0,0 +1,18 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `find_files` from `dist.rs`. + +fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { + let mut found = Vec::with_capacity(files.len()); + + for file in files { + let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); + + if let Some(file_path) = file_path { + found.push(file_path); + } else { + panic!("Could not find '{file}' in {path:?}"); + } + } + + found +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/json_docs.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/json_docs.rs new file mode 100644 index 00000000..f5db8ebc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/json_docs.rs @@ -0,0 +1,39 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `JsonDocs` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct JsonDocs { + pub host: TargetSelection, +} + +impl Step for JsonDocs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs-json").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(JsonDocs { host: run.target }); + } + + /// Builds the `rust-docs-json` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.ensure(crate::core::build_steps::doc::Std::new( + builder.top_stage, + host, + DocumentationFormat::Json, + )); + + let dest = "share/doc/rust/json"; + + let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); + tarball.set_product_name("Rust Documentation In JSON Format"); + tarball.is_preview(true); + tarball.add_bulk_dir(builder.json_doc_out(host), dest); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/make_win_dist.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/make_win_dist.rs new file mode 100644 index 00000000..1ca84c94 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/make_win_dist.rs @@ -0,0 +1,145 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `make_win_dist` from `dist.rs`. + +fn make_win_dist( + rust_root: &Path, + plat_root: &Path, + target: TargetSelection, + builder: &Builder<'_>, +) { + if builder.config.dry_run { + return; + } + + //Ask gcc where it keeps its stuff + let mut cmd = command(builder.cc(target)); + cmd.arg("-print-search-dirs"); + let gcc_out = cmd.run_capture_stdout(builder).stdout(); + + let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); + let mut lib_path = Vec::new(); + + for line in gcc_out.lines() { + let idx = line.find(':').unwrap(); + let key = &line[..idx]; + let trim_chars: &[_] = &[' ', '=']; + let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); + + if key == "programs" { + bin_path.extend(value); + } else if key == "libraries" { + lib_path.extend(value); + } + } + + let compiler = if target == "i686-pc-windows-gnu" { + "i686-w64-mingw32-gcc.exe" + } else if target == "x86_64-pc-windows-gnu" { + "x86_64-w64-mingw32-gcc.exe" + } else { + "gcc.exe" + }; + let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; + let mut rustc_dlls = vec!["libwinpthread-1.dll"]; + if target.starts_with("i686-") { + rustc_dlls.push("libgcc_s_dw2-1.dll"); + } else { + rustc_dlls.push("libgcc_s_seh-1.dll"); + } + + // Libraries necessary to link the windows-gnu toolchains. + // System libraries will be preferred if they are available (see #67429). + let target_libs = [ + //MinGW libs + "libgcc.a", + "libgcc_eh.a", + "libgcc_s.a", + "libm.a", + "libmingw32.a", + "libmingwex.a", + "libstdc++.a", + "libiconv.a", + "libmoldname.a", + "libpthread.a", + //Windows import libs + //This should contain only the set of libraries necessary to link the standard library. + "libadvapi32.a", + "libbcrypt.a", + "libcomctl32.a", + "libcomdlg32.a", + "libcredui.a", + "libcrypt32.a", + "libdbghelp.a", + "libgdi32.a", + "libimagehlp.a", + "libiphlpapi.a", + "libkernel32.a", + "libmsimg32.a", + "libmsvcrt.a", + "libntdll.a", + "libole32.a", + "liboleaut32.a", + "libopengl32.a", + "libpsapi.a", + "librpcrt4.a", + "libsecur32.a", + "libsetupapi.a", + "libshell32.a", + "libsynchronization.a", + "libuser32.a", + "libuserenv.a", + "libuuid.a", + "libwinhttp.a", + "libwinmm.a", + "libwinspool.a", + "libws2_32.a", + "libwsock32.a", + ]; + + //Find mingw artifacts we want to bundle + let target_tools = find_files(&target_tools, &bin_path); + let rustc_dlls = find_files(&rustc_dlls, &bin_path); + let target_libs = find_files(&target_libs, &lib_path); + + // Copy runtime dlls next to rustc.exe + let rust_bin_dir = rust_root.join("bin/"); + fs::create_dir_all(&rust_bin_dir).expect("creating rust_bin_dir failed"); + for src in &rustc_dlls { + builder.copy_link_to_folder(src, &rust_bin_dir); + } + + if builder.config.lld_enabled { + // rust-lld.exe also needs runtime dlls + let rust_target_bin_dir = rust_root.join("lib/rustlib").join(target).join("bin"); + fs::create_dir_all(&rust_target_bin_dir).expect("creating rust_target_bin_dir failed"); + for src in &rustc_dlls { + builder.copy_link_to_folder(src, &rust_target_bin_dir); + } + } + + //Copy platform tools to platform-specific bin directory + let plat_target_bin_self_contained_dir = + plat_root.join("lib/rustlib").join(target).join("bin/self-contained"); + fs::create_dir_all(&plat_target_bin_self_contained_dir) + .expect("creating plat_target_bin_self_contained_dir failed"); + for src in target_tools { + builder.copy_link_to_folder(&src, &plat_target_bin_self_contained_dir); + } + + // Warn windows-gnu users that the bundled GCC cannot compile C files + builder.create( + &plat_target_bin_self_contained_dir.join("GCC-WARNING.txt"), + "gcc.exe contained in this folder cannot be used for compiling C files - it is only + used as a linker. In order to be able to compile projects containing C code use + the GCC provided by MinGW or Cygwin.", + ); + + //Copy platform libs to platform-specific lib directory + let plat_target_lib_self_contained_dir = + plat_root.join("lib/rustlib").join(target).join("lib/self-contained"); + fs::create_dir_all(&plat_target_lib_self_contained_dir) + .expect("creating plat_target_lib_self_contained_dir failed"); + for src in target_libs { + builder.copy_link_to_folder(&src, &plat_target_lib_self_contained_dir); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/mingw.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/mingw.rs new file mode 100644 index 00000000..3708dad0 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/mingw.rs @@ -0,0 +1,42 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Mingw` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Mingw { + pub host: TargetSelection, +} + +impl Step for Mingw { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-mingw") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Mingw { host: run.target }); + } + + /// Builds the `rust-mingw` installer component. + /// + /// This contains all the bits and pieces to run the MinGW Windows targets + /// without any extra installed software (e.g., we bundle gcc, libraries, etc). + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { + return None; + } + + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); + + // The first argument is a "temporary directory" which is just + // thrown away (this contains the runtime DLLs included in the rustc package + // above) and the second argument is where to place all the MinGW components + // (which is what we want). + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, builder); + + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/miri.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/miri.rs new file mode 100644 index 00000000..cf73839f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/miri.rs @@ -0,0 +1,53 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Miri` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Miri { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Miri { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "miri"); + run.alias("miri").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + // This prevents miri from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } + let compiler = self.compiler; + let target = self.target; + + let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() }); + let cargomiri = + builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() }); + + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/pkgname.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/pkgname.rs new file mode 100644 index 00000000..ed97b48b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/pkgname.rs @@ -0,0 +1,6 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `pkgname` from `dist.rs`. + +pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { + format!("{}-{}", component, builder.rust_package_vers()) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/plain_source_tarball.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/plain_source_tarball.rs new file mode 100644 index 00000000..5f175bd7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/plain_source_tarball.rs @@ -0,0 +1,124 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `PlainSourceTarball` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct PlainSourceTarball; + +impl Step for PlainSourceTarball { + /// Produces the location of the tarball generated + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-src").default_condition(builder.config.rust_dist_src) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(PlainSourceTarball); + } + + /// Creates the plain source tarball + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which + // means neither rustup nor rustup-toolchain-install-master know how to download it. + // It also contains symbolic links, unlike other any other dist tarball. + // It's used for distros building rustc from source in a pre-vendored environment. + let mut tarball = Tarball::new(builder, "rustc", "src"); + tarball.permit_symlinks(true); + let plain_dst_src = tarball.image_dir(); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "x.py", + "config.example.toml", + "Cargo.toml", + "Cargo.lock", + ".gitmodules", + ]; + let src_dirs = ["src", "compiler", "library", "tests"]; + + copy_src_dirs(builder, &builder.src, &src_dirs, &[], plain_dst_src); + + // Copy the files normally + for item in &src_files { + builder.copy_link(&builder.src.join(item), &plain_dst_src.join(item)); + } + + // Create the version file + builder.create(&plain_dst_src.join("version"), &builder.rust_version()); + + // Create the files containing git info, to ensure --version outputs the same. + let write_git_info = |info: Option<&Info>, path: &Path| { + if let Some(info) = info { + t!(std::fs::create_dir_all(path)); + channel::write_commit_hash_file(path, &info.sha); + channel::write_commit_info_file(path, info); + } + }; + write_git_info(builder.rust_info().info(), plain_dst_src); + write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo")); + + if builder.config.dist_vendor { + builder.require_and_update_all_submodules(); + + // Vendor all Cargo dependencies + let mut cmd = command(&builder.initial_cargo); + cmd.arg("vendor").arg("--versioned-dirs"); + + for p in default_paths_to_vendor(builder) { + cmd.arg("--sync").arg(p); + } + + cmd + // Will read the libstd Cargo.toml which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .current_dir(plain_dst_src); + + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES + .iter() + .chain(build_helper::RUSTC_PGO_CRATES) + .map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); + for manifest_path in pkgs_for_pgo_training { + cmd.arg("--sync").arg(manifest_path); + } + + let config = cmd.run_capture(builder).stdout(); + + let cargo_config_dir = plain_dst_src.join(".cargo"); + builder.create_dir(&cargo_config_dir); + builder.create(&cargo_config_dir.join("config.toml"), &config); + } + + // Delete extraneous directories + // FIXME: if we're managed by git, we should probably instead ask git if the given path + // is managed by it? + for entry in walkdir::WalkDir::new(tarball.image_dir()) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + { + if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__")) + { + t!(fs::remove_dir_all(entry.path())); + } + } + + tarball.bare() + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rls.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rls.rs new file mode 100644 index 00000000..3aa31ed0 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rls.rs @@ -0,0 +1,44 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Rls` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Rls { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rls { + type Output = Option; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "rls"); + run.alias("rls").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rls { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rls = builder.ensure(tool::Rls { compiler, target, extra_features: Vec::new() }); + + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::Rls); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rust_analyzer.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rust_analyzer.rs new file mode 100644 index 00000000..10ec0a35 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rust_analyzer.rs @@ -0,0 +1,44 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `RustAnalyzer` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "rust-analyzer"); + run.alias("rust-analyzer").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target }); + + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc.rs new file mode 100644 index 00000000..eb0c1696 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc.rs @@ -0,0 +1,165 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Rustc` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub compiler: Compiler, +} + +impl Step for Rustc { + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder + .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); + } + + /// Creates the `rustc` installer component. + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let compiler = self.compiler; + let host = self.compiler.host; + + let tarball = Tarball::new(builder, "rustc", &host.triple); + + // Prepare the rustc "image", what will actually end up getting installed + prepare_image(builder, compiler, tarball.image_dir()); + + // On MinGW we've got a few runtime DLL dependencies that we need to + // include. The first argument to this script is where to put these DLLs + // (the image we're creating), and the second argument is a junk directory + // to ignore all other MinGW stuff the script creates. + // + // On 32-bit MinGW we're always including a DLL which needs some extra + // licenses to distribute. On 64-bit MinGW we don't actually distribute + // anything requiring us to distribute a license, but it's likely the + // install will *also* include the rust-mingw package, which also needs + // licenses, so to be safe we just include it here in all MinGW packages. + if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); + } + + return tarball.generate(); + + fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { + let host = compiler.host; + let src = builder.sysroot(compiler); + + // Copy rustc binary + t!(fs::create_dir_all(image.join("bin"))); + builder.cp_link_r(&src.join("bin"), &image.join("bin")); + + // If enabled, copy rustdoc binary + if builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) + { + let rustdoc = builder.rustdoc(compiler); + builder.install(&rustdoc, &image.join("bin"), 0o755); + } + + if let Some(ra_proc_macro_srv) = builder.ensure_if_default( + tool::RustAnalyzerProcMacroSrv { + compiler: builder.compiler_for( + compiler.stage, + builder.config.build, + compiler.host, + ), + target: compiler.host, + }, + builder.kind, + ) { + builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + } + + let libdir_relative = builder.libdir_relative(compiler); + + // Copy runtime DLLs needed by the compiler + if libdir_relative.to_str() != Some("bin") { + let libdir = builder.rustc_libdir(compiler); + for entry in builder.read_dir(&libdir) { + if is_dylib(&entry.path()) { + // Don't use custom libdir here because ^lib/ will be resolved again + // with installer + builder.install(&entry.path(), &image.join("lib"), 0o644); + } + } + } + + // Copy libLLVM.so to the lib dir as well, if needed. While not + // technically needed by rustc itself it's needed by lots of other + // components like the llvm tools and LLD. LLD is included below and + // tools/LLDB come later, so let's just throw it in the rustc + // component for now. + maybe_install_llvm_runtime(builder, host, image); + + let dst_dir = image.join("lib/rustlib").join(host).join("bin"); + t!(fs::create_dir_all(&dst_dir)); + + // Copy over lld if it's there + if builder.config.lld_enabled { + let src_dir = builder.sysroot_target_bindir(compiler, host); + let rust_lld = exe("rust-lld", compiler.host); + builder.copy_link(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); + let self_contained_lld_src_dir = src_dir.join("gcc-ld"); + let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dst_dir)); + for name in crate::LLD_FILE_NAMES { + builder.copy_link( + &self_contained_lld_src_dir.join(&exe(name, compiler.host)), + &self_contained_lld_dst_dir.join(&exe(name, compiler.host)), + ); + } + } + + if builder.config.llvm_enabled(compiler.host) && builder.config.llvm_tools_enabled { + let src_dir = builder.sysroot_target_bindir(compiler, host); + let llvm_objcopy = exe("llvm-objcopy", compiler.host); + let rust_objcopy = exe("rust-objcopy", compiler.host); + builder.copy_link(&src_dir.join(&llvm_objcopy), &dst_dir.join(&rust_objcopy)); + } + + if builder.tool_enabled("wasm-component-ld") { + let src_dir = builder.sysroot_target_bindir(compiler, host); + let ld = exe("wasm-component-ld", compiler.host); + builder.copy_link(&src_dir.join(&ld), &dst_dir.join(&ld)); + } + + // Man pages + t!(fs::create_dir_all(image.join("share/man/man1"))); + let man_src = builder.src.join("src/doc/man"); + let man_dst = image.join("share/man/man1"); + + // don't use our `bootstrap::{copy_internal, cp_r}`, because those try + // to hardlink, and we don't want to edit the source templates + for file_entry in builder.read_dir(&man_src) { + let page_src = file_entry.path(); + let page_dst = man_dst.join(file_entry.file_name().unwrap()); + let src_text = t!(std::fs::read_to_string(&page_src)); + let new_text = src_text.replace("", &builder.version); + t!(std::fs::write(&page_dst, &new_text)); + t!(fs::copy(&page_src, &page_dst)); + } + + // Debugger scripts + builder.ensure(DebuggerScripts { sysroot: image.to_owned(), host }); + + // Misc license info + let cp = |file: &str| { + builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); + }; + cp("COPYRIGHT"); + cp("LICENSE-APACHE"); + cp("LICENSE-MIT"); + cp("README.md"); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_dev.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_dev.rs new file mode 100644 index 00000000..19a7adf3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_dev.rs @@ -0,0 +1,65 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `RustcDev` from `dist.rs`. + +/// Tarball containing the compiler that gets downloaded and used by +/// `rust.download-rustc`. +/// +/// (Don't confuse this with [`RustDev`], without the `c`!) +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct RustcDev { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustcDev { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDev { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Rustc::new(compiler, target)); + + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); + + let src_files = &["Cargo.lock"]; + // This is the reduced set of paths which will become the rustc-dev component + // (essentially the compiler crates and all of their path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); + } + + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_docs.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_docs.rs new file mode 100644 index 00000000..da9f59ec --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustc_docs.rs @@ -0,0 +1,33 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `RustcDocs` from `dist.rs`. + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct RustcDocs { + pub host: TargetSelection, +} + +impl Step for RustcDocs { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-docs").default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDocs { host: run.target }); + } + + /// Builds the `rustc-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_bulk_dir(builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustfmt.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustfmt.rs new file mode 100644 index 00000000..f28d8d78 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/rustfmt.rs @@ -0,0 +1,47 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Rustfmt` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Rustfmt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rustfmt { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(run.builder, "rustfmt"); + run.alias("rustfmt").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rustfmt = + builder.ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }); + let cargofmt = + builder.ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }); + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/should_build_extended_tool.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/should_build_extended_tool.rs new file mode 100644 index 00000000..9c8cf61a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/should_build_extended_tool.rs @@ -0,0 +1,9 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `should_build_extended_tool` from `dist.rs`. + +fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { + if !builder.config.extended { + return false; + } + builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/skip_host_target_lib.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/skip_host_target_lib.rs new file mode 100644 index 00000000..f98e174f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/skip_host_target_lib.rs @@ -0,0 +1,14 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `skip_host_target_lib` from `dist.rs`. + +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info(" skipping, not a build host"); + true + } else { + false + } +} + diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/src.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/src.rs new file mode 100644 index 00000000..b601ca96 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/src.rs @@ -0,0 +1,59 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Src` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Src; + +impl Step for Src { + /// The output path of the src installer tarball + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-src") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src); + } + + /// Creates the `rust-src` installer component + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + if !builder.config.dry_run { + builder.require_submodule("src/llvm-project", None); + } + + let tarball = Tarball::new_targetless(builder, "rust-src"); + + // A lot of tools expect the rust-src component to be entirely in this directory, so if you + // change that (e.g. by adding another directory `lib/rustlib/src/foo` or + // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions + // and fix them... + // + // NOTE: if you update the paths here, you also should update the "virtual" path + // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); + + // This is the reduced set of paths which will become the rust-src component + // (essentially libstd and all of its path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["library", "src/llvm-project/libunwind"], + &[ + // not needed and contains symlinks which rustup currently + // chokes on when unpacking. + "library/backtrace/crates", + // these are 30MB combined and aren't necessary for building + // the standard library. + "library/stdarch/Cargo.toml", + "library/stdarch/crates/stdarch-verify", + "library/stdarch/crates/intrinsic-test", + ], + &dst_src, + ); + + tarball.generate() + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/std.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/std.rs new file mode 100644 index 00000000..890610d0 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/std.rs @@ -0,0 +1,49 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `Std` from `dist.rs`. + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct Std { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Std { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-std") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Std::new(compiler, target)); + + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + verify_uefi_rlib_format(builder, target, &stamp); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); + + Some(tarball.generate()) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/tmpdir.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/tmpdir.rs new file mode 100644 index 00000000..104d6fbb --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/tmpdir.rs @@ -0,0 +1,6 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `tmpdir` from `dist.rs`. + +pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("tmp/dist") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/dist_modules/verify_uefi_rlib_format.rs b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/verify_uefi_rlib_format.rs new file mode 100644 index 00000000..cd6c7b94 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/dist_modules/verify_uefi_rlib_format.rs @@ -0,0 +1,35 @@ +//! This file was automatically generated by a refactoring script. +//! It contains the definition of `verify_uefi_rlib_format` from `dist.rs`. + +/// Check that all objects in rlibs for UEFI targets are COFF. This +/// ensures that the C compiler isn't producing ELF objects, which would +/// not link correctly with the COFF objects. +fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { + if !target.ends_with("-uefi") { + return; + } + + for (path, _) in builder.read_stamp_file(stamp) { + if path.extension() != Some(OsStr::new("rlib")) { + continue; + } + + let data = t!(fs::read(&path)); + let data = data.as_slice(); + let archive = t!(ArchiveFile::parse(data)); + for member in archive.members() { + let member = t!(member); + let member_data = t!(member.data(data)); + + let is_coff = match object::File::parse(member_data) { + Ok(member_file) => member_file.format() == BinaryFormat::Coff, + Err(_) => false, + }; + + if !is_coff { + let member_name = String::from_utf8_lossy(member.name()); + panic!("member {} in {} is not COFF", member_name, path.display()); + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/doc.rs b/standalonex/src/bootstrap/src/core/build_steps/doc.rs index 8a9321f8..5ed22076 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/doc.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/doc.rs @@ -150,7 +150,7 @@ impl Step for RustbookSrc

{ let index = out.join("index.html"); let rustbook = builder.tool_exe(Tool::Rustbook); - if !builder.config.dry_run() + if !builder.config.dry_run && (!up_to_date(&src, &index) || !up_to_date(&rustbook, &index)) { builder.info(&format!("Rustbook ({target}) - {name}")); @@ -380,8 +380,8 @@ impl Step for Standalone { && up_to_date(&footer, &html) && up_to_date(&favicon, &html) && up_to_date(&full_toc, &html) - && (builder.config.dry_run() || up_to_date(&version_info, &html)) - && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) + && (builder.config.dry_run || up_to_date(&version_info, &html)) + && (builder.config.dry_run || up_to_date(&rustdoc, &html)) { continue; } @@ -478,7 +478,7 @@ impl Step for Releases { || !up_to_date(&footer, &html) || !up_to_date(&favicon, &html) || !up_to_date(&full_toc, &html) - || !(builder.config.dry_run() + || !(builder.config.dry_run || up_to_date(&version_info, &html) || up_to_date(&rustdoc, &html)) { @@ -546,7 +546,7 @@ impl Step for SharedAssets { let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); let version_info = out.join("version_info.html"); - if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { + if !builder.config.dry_run && !up_to_date(&version_input, &version_info) { let info = t!(fs::read_to_string(&version_input)) .replace("VERSION", &builder.rust_release()) .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) @@ -882,7 +882,7 @@ impl Step for Rustc { cargo.into_cmd().run(builder); - if !builder.config.dry_run() { + if !builder.config.dry_run { // Sanity check on linked compiler crates for krate in &*self.crates { let dir_name = krate.replace('-', "_"); @@ -1014,7 +1014,7 @@ macro_rules! tool_doc { let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); cargo.into_cmd().run(builder); - if !builder.config.dry_run() { + if !builder.config.dry_run { // Sanity check on linked doc directories $(for krate in $crates { let dir_name = krate.replace("-", "_"); @@ -1147,7 +1147,7 @@ impl Step for UnstableBookGen { } fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { - if config.dry_run() { + if config.dry_run { return; } if let Ok(m) = fs::symlink_metadata(link) { diff --git a/standalonex/src/bootstrap/src/core/build_steps/format.rs b/standalonex/src/bootstrap/src/core/build_steps/format.rs index 5ca4321d..4dd57e4f 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/format.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/format.rs @@ -124,7 +124,7 @@ pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) { ); crate::exit!(1); }; - if build.config.dry_run() { + if build.config.dry_run { return; } diff --git a/standalonex/src/bootstrap/src/core/build_steps/gcc.rs b/standalonex/src/bootstrap/src/core/build_steps/gcc.rs index b950bec1..4c0a158b 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/gcc.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/gcc.rs @@ -108,7 +108,7 @@ impl Step for Gcc { let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); - if builder.config.dry_run() { + if builder.config.dry_run { return true; } diff --git a/standalonex/src/bootstrap/src/core/build_steps/llvm.rs b/standalonex/src/bootstrap/src/core/build_steps/llvm.rs index 9734a0dc..573c2cce 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/llvm.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/llvm.rs @@ -509,7 +509,7 @@ impl Step for Llvm { if target != builder.config.build { let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if !builder.config.dry_run() { + if !builder.config.dry_run { let llvm_bindir = command(&llvm_config).arg("--bindir").run_capture_stdout(builder).stdout(); let host_bin = Path::new(llvm_bindir.trim()); @@ -524,7 +524,7 @@ impl Step for Llvm { if builder.config.llvm_clang { let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); - if !builder.config.dry_run() && !clang_tblgen.exists() { + if !builder.config.dry_run && !clang_tblgen.exists() { panic!("unable to find {}", clang_tblgen.display()); } cfg.define("CLANG_TABLEGEN", clang_tblgen); @@ -553,7 +553,7 @@ impl Step for Llvm { cfg.define(key, val); } - if builder.config.dry_run() { + if builder.config.dry_run { return res; } @@ -615,7 +615,7 @@ impl Step for Llvm { } fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -915,7 +915,7 @@ impl Step for Enzyme { "src/tools/enzyme", Some("The Enzyme sources are required for autodiff."), ); - if builder.config.dry_run() { + if builder.config.dry_run { let out_dir = builder.enzyme_out(self.target); return out_dir; } @@ -1005,7 +1005,7 @@ impl Step for Lld { /// Compile LLD for `target`. fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { + if builder.config.dry_run { return PathBuf::from("lld-out-dir-test-gen"); } let target = self.target; @@ -1138,7 +1138,7 @@ impl Step for Sanitizers { } let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if builder.config.dry_run() { + if builder.config.dry_run { return runtimes; } @@ -1312,7 +1312,7 @@ impl Step for CrtBeginEnd { let out_dir = builder.native_dir(self.target).join("crt"); - if builder.config.dry_run() { + if builder.config.dry_run { return out_dir; } @@ -1386,7 +1386,7 @@ impl Step for Libunwind { Some("The LLVM sources are required for libunwind."), ); - if builder.config.dry_run() { + if builder.config.dry_run { return PathBuf::new(); } diff --git a/standalonex/src/bootstrap/src/core/build_steps/mod.rs b/standalonex/src/bootstrap/src/core/build_steps/mod.rs index fcb6abea..07065fb9 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/mod.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/mod.rs @@ -1,19 +1,21 @@ -pub(crate) mod check; -pub(crate) mod clean; -pub(crate) mod clippy; -pub(crate) mod compile; -pub(crate) mod dist; -pub(crate) mod doc; -pub(crate) mod format; -pub(crate) mod gcc; -pub(crate) mod install; -pub(crate) mod llvm; -pub(crate) mod perf; -pub(crate) mod run; -pub(crate) mod setup; -pub(crate) mod suggest; -pub(crate) mod synthetic_targets; -pub(crate) mod test; -pub(crate) mod tool; -pub(crate) mod toolstate; -pub(crate) mod vendor; +pub mod check; +pub mod clean; +pub mod clippy; +pub mod compile; +pub mod compile_modules; +pub mod dist; +pub mod doc; +pub mod format; +pub mod gcc; +pub mod install; +pub mod llvm; +pub mod perf; +pub mod run; +pub mod setup; +pub mod suggest; +pub mod synthetic_targets; +pub mod test; +pub mod test_split; +pub mod tool; +pub mod toolstate; +pub mod vendor; diff --git a/standalonex/src/bootstrap/src/core/build_steps/run.rs b/standalonex/src/bootstrap/src/core/build_steps/run.rs index a6dff7fd..7fb9a783 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/run.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/run.rs @@ -1,7 +1,8 @@ -//! Build-and-run steps for in-repo tools -//! -//! A bit of a hodge-podge as e.g. if a tool's a test fixture it should be in `build_steps::test`. -//! If it can be reached from `./x.py run` it can go here. +use crate::prelude::*; +// Build-and-run steps for in-repo tools +// +// A bit of a hodge-podge as e.g. if a tool's a test fixture it should be in `build_steps::test`. +// If it can be reached from `./x.py run` it can go here. use std::path::PathBuf; diff --git a/standalonex/src/bootstrap/src/core/build_steps/rustc_step_common.rs b/standalonex/src/bootstrap/src/core/build_steps/rustc_step_common.rs new file mode 100644 index 00000000..e893dbcd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/rustc_step_common.rs @@ -0,0 +1,5 @@ +use crate::builder::ShouldRun; + +pub fn rustc_should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main").path("compiler") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/setup.rs b/standalonex/src/bootstrap/src/core/build_steps/setup.rs index 03cc2892..1414d914 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/setup.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/setup.rs @@ -115,7 +115,7 @@ impl Step for Profile { } fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { + if run.builder.config.dry_run { return; } @@ -239,7 +239,7 @@ impl Step for Link { } fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { + if run.builder.config.dry_run { return; } if let [cmd] = &run.paths[..] { @@ -251,7 +251,7 @@ impl Step for Link { fn run(self, builder: &Builder<'_>) -> Self::Output { let config = &builder.config; - if config.dry_run() { + if config.dry_run { return; } @@ -263,7 +263,7 @@ impl Step for Link { let stage_path = ["build", config.build.rustc_target_arg(), "stage1"].join(MAIN_SEPARATOR_STR); - if stage_dir_exists(&stage_path[..]) && !config.dry_run() { + if stage_dir_exists(&stage_path[..]) && !config.dry_run { attempt_toolchain_link(builder, &stage_path[..]); } } @@ -456,7 +456,7 @@ impl Step for Hook { run.alias("hook") } fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { + if run.builder.config.dry_run { return; } if let [cmd] = &run.paths[..] { @@ -467,7 +467,7 @@ impl Step for Hook { } fn run(self, builder: &Builder<'_>) -> Self::Output { let config = &builder.config; - if config.dry_run() { + if config.dry_run { return; } t!(install_git_hook_maybe(builder, config)); @@ -633,7 +633,7 @@ impl Step for Editor { } fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { + if run.builder.config.dry_run { return; } if let [cmd] = &run.paths[..] { @@ -645,7 +645,7 @@ impl Step for Editor { fn run(self, builder: &Builder<'_>) -> Self::Output { let config = &builder.config; - if config.dry_run() { + if config.dry_run { return; } match EditorKind::prompt_user() { @@ -677,7 +677,7 @@ fn create_editor_settings_maybe(config: &Config, editor: EditorKind) -> io::Resu if let Ok(current) = fs::read_to_string(&settings_path) { let mut hasher = sha2::Sha256::new(); hasher.update(¤t); - let hash = hex_encode(hasher.finalize().as_slice()); + let hash = hex_encode(hasher.finalize()); if hash == *current_hash { return Ok(true); } else if historical_hashes.contains(&hash.as_str()) { diff --git a/standalonex/src/bootstrap/src/core/build_steps/synthetic_targets.rs b/standalonex/src/bootstrap/src/core/build_steps/synthetic_targets.rs index 477ff955..9a5da07a 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/synthetic_targets.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -51,7 +51,7 @@ fn create_synthetic_target( let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - if builder.config.dry_run() { + if builder.config.dry_run { std::fs::write(&path, b"dry run\n").unwrap(); return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); } diff --git a/standalonex/src/bootstrap/src/core/build_steps/test.rs b/standalonex/src/bootstrap/src/core/build_steps/test.rs index dcea9f5f..392fceef 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/test.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/test.rs @@ -1,7 +1,9 @@ -//! Build-and-run steps for `./x.py test` test fixtures -//! -//! `./x.py test` (aka [`Kind::Test`]) is currently allowed to reach build steps in other modules. -//! However, this contains ~all test parts we expect people to be able to build and run locally. +use super::test_split::*; +use crate::prelude::*; +// Build-and-run steps for `./x.py test` test fixtures +// +// `./x.py test` (aka [`Kind::Test`]) is currently allowed to reach build steps in other modules. +// However, this contains ~all test parts we expect people to be able to build and run locally. use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; @@ -18,7 +20,8 @@ use crate::core::builder::{ self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, crate_description, }; use crate::core::config::TargetSelection; -use crate::core::config::flags::{Subcommand, get_completion}; +use crate::core::config::flags::get_completion; +use crate::Subcommand; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, LldThreads, add_link_lib_path, add_rustdoc_cargo_linker_args, dylib_path, dylib_path_var, @@ -27,3586 +30,273 @@ use crate::utils::helpers::{ use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; use crate::{CLang, DocTests, GitRepo, Mode, envify}; -const ADB_TEST_DIR: &str = "/data/local/tmp/work"; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateBootstrap { - path: PathBuf, - host: TargetSelection, -} - -impl Step for CrateBootstrap { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/jsondoclint") - .path("src/tools/suggest-tests") - .path("src/tools/replace-version-placeholder") - .alias("tidyselftest") - } - - fn make_run(run: RunConfig<'_>) { - for path in run.paths { - let path = path.assert_single_path().path.clone(); - run.builder.ensure(CrateBootstrap { host: run.target, path }); - } - } - - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let mut path = self.path.to_str().unwrap(); - if path == "tidyselftest" { - path = "src/tools/tidy"; - } - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - Kind::Test, - path, - SourceType::InTree, - &[], - ); - let crate_name = path.rsplit_once('/').unwrap().1; - run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); - } +macro_rules! default_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); + }; } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Linkcheck { - host: TargetSelection, +macro_rules! default_test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, + compare_mode: $compare_mode:expr }) => { + test_with_compare_mode!($name { + path: $path, + mode: $mode, + suite: $suite, + default: true, + host: false, + compare_mode: $compare_mode + }); + }; } -impl Step for Linkcheck { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will verify the validity of all our links in the - /// documentation to ensure we don't have a bunch of dead ones. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let hosts = &builder.hosts; - let targets = &builder.targets; - - // if we have different hosts and targets, some things may be built for - // the host (e.g. rustc) and others for the target (e.g. std). The - // documentation built for each will contain broken links to - // docs built for the other platform (e.g. rustc linking to cargo) - if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { - panic!( - "Linkcheck currently does not support builds with different hosts and targets. -You can skip linkcheck with --skip src/tools/linkchecker" - ); - } - - builder.info(&format!("Linkcheck ({host})")); - - // Test the linkchecker itself. - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - Kind::Test, - "src/tools/linkchecker", - SourceType::InTree, - &[], - ); - run_cargo_test( - cargo, - &[], - &[], - "linkchecker", - "linkchecker self tests", - compiler, - bootstrap_host, - builder, - ); - - if builder.doc_tests == DocTests::No { - return; - } - - // Build all the default documentation. - builder.default_doc(&[]); - - // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. - let linkchecker = builder.tool_cmd(Tool::Linkchecker); - - // Run the linkchecker. - let _guard = - builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); - let _time = helpers::timeit(builder); - linkchecker.delay_failure().arg(builder.out.join(host).join("doc")).run(builder); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.path("src/tools/linkchecker"); - run.default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Linkcheck { host: run.target }); - } +macro_rules! host_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); + }; } -fn check_if_tidy_is_installed(builder: &Builder<'_>) -> bool { - command("tidy").allow_failure().arg("--version").run_capture_stdout(builder).is_success() +macro_rules! test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr }) => { + crate::test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: None + }); + }; } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HtmlCheck { - target: TargetSelection, +macro_rules! test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr, compare_mode: $compare_mode:expr }) => { + crate::test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: Some($compare_mode) + }); + }; } -impl Step for HtmlCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.path("src/tools/html-checker"); - run.lazy_default_condition(Box::new(|| check_if_tidy_is_installed(builder))) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(HtmlCheck { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - if !check_if_tidy_is_installed(builder) { - eprintln!("not running HTML-check tool because `tidy` is missing"); - eprintln!( - "You need the HTML tidy tool https://www.html-tidy.org/, this tool is *not* part of the rust project and needs to be installed separately, for example via your package manager." - ); - panic!("Cannot run html-check tests"); +macro_rules! test_definitions { + ($name:ident { + path: $path:expr, + mode: $mode:expr, + suite: $suite:expr, + default: $default:expr, + host: $host:expr, + compare_mode: $compare_mode:expr + }) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, } - // Ensure that a few different kinds of documentation are available. - builder.default_doc(&[]); - builder.ensure(crate::core::build_steps::doc::Rustc::new( - builder.top_stage, - self.target, - builder, - )); - - builder - .tool_cmd(Tool::HtmlChecker) - .delay_failure() - .arg(builder.doc_out(self.target)) - .run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Cargotest { - stage: u32, - host: TargetSelection, -} - -impl Step for Cargotest { - type Output = (); - const ONLY_HOSTS: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargotest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); - } + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $host; - /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will check out a few Rust projects and run `cargo - /// test` to ensure that we don't regress the test suites there. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); - builder.ensure(compile::Rustc::new(compiler, compiler.host)); - let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path($path) + } - // Note that this is a short, cryptic, and not scoped directory name. This - // is currently to minimize the length of path on Windows where we otherwise - // quickly run into path name limit constraints. - let out_dir = builder.out.join("ct"); - t!(fs::create_dir_all(&out_dir)); + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - let _time = helpers::timeit(builder); - let mut cmd = builder.tool_cmd(Tool::CargoTest); - cmd.arg(&cargo) - .arg(&out_dir) - .args(builder.config.test_args()) - .env("RUSTC", builder.rustc(compiler)) - .env("RUSTDOC", builder.rustdoc(compiler)); - add_rustdoc_cargo_linker_args(&mut cmd, builder, compiler.host, LldThreads::No); - cmd.delay_failure().run(builder); - } -} + run.builder.ensure($name { compiler, target: run.target }); + } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Cargo { - stage: u32, - host: TargetSelection, + fn run(self, builder: &Builder<'_>) { + builder.ensure(crate::compiletest::Compiletest { + compiler: self.compiler, + target: self.target, + mode: $mode, + suite: $suite, + path: $path, + compare_mode: $compare_mode, + }) + } + } + }; } -impl Step for Cargo { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargo") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for `cargo` packaged with Rust. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); +/// Declares an alias for running the [`Coverage`] tests in only one mode. +/// Adapted from [`test_definitions`]. +macro_rules! coverage_test_alias { + ($name:ident { + alias_and_mode: $alias_and_mode:expr, // &'static str + default: $default:expr, // bool + only_hosts: $only_hosts:expr $(,)? // bool + }) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } - builder.ensure(tool::Cargo { compiler, target: self.host }); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - self.host, - Kind::Test, - "src/tools/cargo", - SourceType::Submodule, - &[], - ); + impl $name { + const MODE: &'static str = $alias_and_mode; + } - // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` - let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $only_hosts; - // Don't run cross-compile tests, we may not have cross-compiled libstd libs - // available. - cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); - // Forcibly disable tests using nightly features since any changes to - // those features won't be able to land. - cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); - cargo.env("PATH", path_for_cargo(builder, compiler)); + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Register the mode name as a command-line alias. + // This enables `x test coverage-map` and `x test coverage-run`. + run.alias($alias_and_mode) + } - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: vec!["cargo".into()], - target: self.host.triple.to_string(), - host: self.host.triple.to_string(), - stage: self.stage, - }, - builder, - ); + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - let _time = helpers::timeit(builder); - add_flags_and_try_run_tests(builder, &mut cargo); - } -} + run.builder.ensure($name { compiler, target: run.target }); + } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - stage: u32, - host: TargetSelection, + fn run(self, builder: &Builder<'_>) { + crate::coverage::Coverage::run_coverage_tests(builder, self.compiler, self.target, Self::MODE); + } + } + }; } -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-analyzer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-analyzer - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, - // but we do need the standard library to be present. - builder.ensure(compile::Rustc::new(compiler, host)); +macro_rules! test_book { + ($( + $name:ident, $path:expr, $book_name:expr, + default=$default:expr + $(,submodules = $submodules:expr)? + ; + )+) => { + $( + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + } - let workspace_path = "src/tools/rust-analyzer"; - // until the whole RA test suite runs on `i686`, we only run - // `proc-macro-srv` tests - let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - Kind::Test, - crate_path, - SourceType::InTree, - &["in-rust-tree".to_owned()], - ); - cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = true; - let dir = builder.src.join(workspace_path); - // needed by rust-analyzer to find its own text fixtures, cf. - // https://github.com/rust-analyzer/expect-test/issues/33 - cargo.env("CARGO_WORKSPACE_DIR", &dir); + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } - // RA's test suite tries to write to the source directory, that can't - // work in Rust CI - cargo.env("SKIP_SLOW_TESTS", "1"); + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } - cargo.add_rustc_lib_path(builder); - run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); + fn run(self, builder: &Builder<'_>) { + $( + for submodule in $submodules { + builder.require_submodule(submodule, None); + } + )* + builder.ensure(crate::book_test::BookTest { + compiler: self.compiler, + path: PathBuf::from($path), + name: $book_name, + is_ext_doc: !$default, + }); + } + } + )+ } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Rustfmt { - stage: u32, - host: TargetSelection, -} - -impl Step for Rustfmt { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustfmt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rustfmt. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder.ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - Kind::Test, - "src/tools/rustfmt", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - cargo.env("RUSTFMT_TEST_DIR", dir); - - cargo.add_rustc_lib_path(builder); - - run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); - } -} +test_book!( + Nomicon, "src/doc/nomicon", "nomicon", default=false, submodules=["src/doc/nomicon"]; + Reference, "src/doc/reference", "reference", default=false, submodules=["src/doc/reference"]; + RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; + RustcBook, "src/doc/rustc", "rustc", default=true; + RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false, submodules=["src/doc/rust-by-example"]; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false, submodules=["src/doc/embedded-book"]; + TheBook, "src/doc/book", "book", default=false, submodules=["src/doc/book"]; + UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; + EditionGuide, "src/doc/edition-guide", "edition-guide", default=false, submodules=["src/doc/edition-guide"]; +); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - target: TargetSelection, -} +default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); -impl Miri { - /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. - pub fn build_miri_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - ) -> PathBuf { - let miri_sysroot = builder.out.join(compiler.host).join("miri-sysroot"); - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::Submodule, - target, - Kind::MiriSetup, - ); +default_test!(Crashes { path: "tests/crashes", mode: "crashes", suite: "crashes" }); - // Tell `cargo miri setup` where to find the sources. - cargo.env("MIRI_LIB_SRC", builder.src.join("library")); - // Tell it where to put the sysroot. - cargo.env("MIRI_SYSROOT", &miri_sysroot); +default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); - let mut cargo = BootstrapCommand::from(cargo); - let _guard = - builder.msg(Kind::Build, compiler.stage, "miri sysroot", compiler.host, target); - cargo.run(builder); +default_test!(CodegenUnits { + path: "tests/codegen-units", + mode: "codegen-units", + suite: "codegen-units" +}); - // # Determine where Miri put its sysroot. - // To this end, we run `cargo miri setup --print-sysroot` and capture the output. - // (We do this separately from the above so that when the setup actually - // happens we get some output.) - // We re-use the `cargo` from above. - cargo.arg("--print-sysroot"); +default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); - builder.verbose(|| println!("running: {cargo:?}")); - let stdout = cargo.run_capture_stdout(builder).stdout(); - // Output is "\n". - let sysroot = stdout.trim_end(); - builder.verbose(|| println!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); - PathBuf::from(sysroot) - } -} +default_test_with_compare_mode!(Debuginfo { + path: "tests/debuginfo", + mode: "debuginfo", + suite: "debuginfo", + compare_mode: "split-dwarf" +}); -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; +host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } +host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); +host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { target: run.target }); - } +host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); - /// Runs `cargo test` for miri. - fn run(self, builder: &Builder<'_>) { - let host = builder.build.build; - let target = self.target; - let stage = builder.top_stage; - if stage == 0 { - eprintln!("miri cannot be tested at stage 0"); - std::process::exit(1); - } +host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); - // This compiler runs on the host, we'll just use it for the target. - let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); +/// Special-handling is needed for `run-make`, so don't use `default_test` for defining `RunMake` +/// tests. +default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); - // Build our tools. - let miri = builder.ensure(tool::Miri { - compiler: host_compiler, - target: host, - extra_features: Vec::new(), - }); - // the ui tests also assume cargo-miri has been built - builder.ensure(tool::CargoMiri { - compiler: host_compiler, - target: host, - extra_features: Vec::new(), - }); - - // We also need sysroots, for Miri and for the host (the latter for build scripts). - // This is for the tests so everything is done with the target compiler. - let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target); - builder.ensure(compile::Std::new(target_compiler, host)); - let host_sysroot = builder.sysroot(target_compiler); - - // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when - // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. - if !builder.config.dry_run() { - let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); - // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see - // ). - // We can hence use that directly as a signal to clear the ui test dir. - builder.clear_if_dirty(&ui_test_dep_dir, &miri_sysroot); - } - - // Run `cargo test`. - // This is with the Miri crate, so it uses the host compiler. - let mut cargo = tool::prepare_tool_cargo( - builder, - host_compiler, - Mode::ToolRustc, - host, - Kind::Test, - "src/tools/miri", - SourceType::InTree, - &[], - ); - - cargo.add_rustc_lib_path(builder); - - // We can NOT use `run_cargo_test` since Miri's integration tests do not use the usual test - // harness and therefore do not understand the flags added by `add_flags_and_try_run_test`. - let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", host_compiler, host, builder); - - // miri tests need to know about the stage sysroot - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", &host_sysroot); - cargo.env("MIRI", &miri); - - // Set the target. - cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); - - { - let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "miri", host, target); - let _time = helpers::timeit(builder); - cargo.run(builder); - } - - // Run it again for mir-opt-level 4 to catch some miscompilations. - if builder.config.test_args().is_empty() { - cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); - // Optimizations can change backtraces - cargo.env("MIRI_SKIP_UI_CHECKS", "1"); - // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible - cargo.env_remove("RUSTC_BLESS"); - // Optimizations can change error locations and remove UB so don't run `fail` tests. - cargo.args(["tests/pass", "tests/panic"]); - - { - let _guard = builder.msg_sysroot_tool( - Kind::Test, - stage, - "miri (mir-opt-level 4)", - host, - target, - ); - let _time = helpers::timeit(builder); - cargo.run(builder); - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CargoMiri { - target: TargetSelection, -} - -impl Step for CargoMiri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri/cargo-miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CargoMiri { target: run.target }); - } - - /// Tests `cargo miri test`. - fn run(self, builder: &Builder<'_>) { - let host = builder.build.build; - let target = self.target; - let stage = builder.top_stage; - if stage == 0 { - eprintln!("cargo-miri cannot be tested at stage 0"); - std::process::exit(1); - } - - // This compiler runs on the host, we'll just use it for the target. - let compiler = builder.compiler(stage, host); - - // Run `cargo miri test`. - // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures - // that we get the desired output), but that is sufficient to make sure that the libtest harness - // itself executes properly under Miri, and that all the logic in `cargo-miri` does not explode. - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, // it's unclear what to use here, we're not building anything just doing a smoke test! - target, - Kind::MiriTest, - "src/tools/miri/test-cargo-miri", - SourceType::Submodule, - &[], - ); - - // We're not using `prepare_cargo_test` so we have to do this ourselves. - // (We're not using that as the test-cargo-miri crate is not known to bootstrap.) - match builder.doc_tests { - DocTests::Yes => {} - DocTests::No => { - cargo.args(["--lib", "--bins", "--examples", "--tests", "--benches"]); - } - DocTests::Only => { - cargo.arg("--doc"); - } - } - - // Finally, pass test-args and run everything. - cargo.arg("--").args(builder.config.test_args()); - let mut cargo = BootstrapCommand::from(cargo); - { - let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "cargo-miri", host, target); - let _time = helpers::timeit(builder); - cargo.run(builder); - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CompiletestTest { - host: TargetSelection, -} - -impl Step for CompiletestTest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/compiletest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CompiletestTest { host: run.target }); - } - - /// Runs `cargo test` for compiletest. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(builder.top_stage, host); - - // We need `ToolStd` for the locally-built sysroot because - // compiletest uses unstable features of the `test` crate. - builder.ensure(compile::Std::new(compiler, host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - // compiletest uses libtest internals; make it use the in-tree std to make sure it never breaks - // when std sources change. - Mode::ToolStd, - host, - Kind::Test, - "src/tools/compiletest", - SourceType::InTree, - &[], - ); - cargo.allow_features("test"); - run_cargo_test( - cargo, - &[], - &[], - "compiletest", - "compiletest self test", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Clippy { - stage: u32, - host: TargetSelection, -} - -impl Step for Clippy { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/clippy") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for clippy. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder.ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - Kind::Test, - "src/tools/clippy", - SourceType::InTree, - &[], - ); - - cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); - cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); - let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); - cargo.env("HOST_LIBS", host_libs); - - cargo.add_rustc_lib_path(builder); - let cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); - - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); - - // Clippy reports errors if it blessed the outputs - if cargo.allow_failure().run(builder) { - // The tests succeeded; nothing to do. - return; - } - - if !builder.config.cmd.bless() { - crate::exit!(1); - } - } -} - -fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { - // Configure PATH to find the right rustc. NB. we have to use PATH - // and not RUSTC because the Cargo test suite has tests that will - // fail if rustc is not spelled `rustc`. - let path = builder.sysroot(compiler).join("bin"); - let old_path = env::var_os("PATH").unwrap_or_default(); - env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustdocTheme { - pub compiler: Compiler, -} - -impl Step for RustdocTheme { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc-themes") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.target); - - run.builder.ensure(RustdocTheme { compiler }); - } - - fn run(self, builder: &Builder<'_>) { - let rustdoc = builder.bootstrap_out.join("rustdoc"); - let mut cmd = builder.tool_cmd(Tool::RustdocTheme); - cmd.arg(rustdoc.to_str().unwrap()) - .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) - .env("RUSTC_STAGE", self.compiler.stage.to_string()) - .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) - .env("RUSTDOC_LIBDIR", builder.sysroot_target_libdir(self.compiler, self.compiler.host)) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - cmd.args(linker_args(builder, self.compiler.host, LldThreads::No)); - - cmd.delay_failure().run(builder); - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSStd { - pub target: TargetSelection, -} - -impl Step for RustdocJSStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js-std").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustdocJSStd { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let nodejs = - builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); - let mut command = command(nodejs); - command - .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) - .arg("--crate-name") - .arg("std") - .arg("--resource-suffix") - .arg(&builder.version) - .arg("--doc-folder") - .arg(builder.doc_out(self.target)) - .arg("--test-folder") - .arg(builder.src.join("tests/rustdoc-js-std")); - for path in &builder.paths { - if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) - { - if !p.ends_with(".js") { - eprintln!("A non-js file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-js-std tests"); - } - command.arg("--test-file").arg(path); - } - } - builder.ensure(crate::core::build_steps::doc::Std::new( - builder.top_stage, - self.target, - DocumentationFormat::Html, - )); - let _guard = builder.msg( - Kind::Test, - builder.top_stage, - "rustdoc-js-std", - builder.config.build, - self.target, - ); - command.run(builder); - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSNotStd { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocJSNotStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "js-doc-test", - suite: "rustdoc-js", - path: "tests/rustdoc-js", - compare_mode: None, - }); - } -} - -fn get_browser_ui_test_version_inner( - builder: &Builder<'_>, - npm: &Path, - global: bool, -) -> Option { - let mut command = command(npm); - command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); - if global { - command.arg("--global"); - } - let lines = command.allow_failure().run_capture(builder).stdout(); - lines - .lines() - .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) - .map(|v| v.to_owned()) -} - -fn get_browser_ui_test_version(builder: &Builder<'_>, npm: &Path) -> Option { - get_browser_ui_test_version_inner(builder, npm, false) - .or_else(|| get_browser_ui_test_version_inner(builder, npm, true)) -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustdocGUI { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocGUI { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.suite_path("tests/rustdoc-gui"); - run.lazy_default_condition(Box::new(move || { - builder.config.nodejs.is_some() - && builder.doc_tests != DocTests::Only - && builder - .config - .npm - .as_ref() - .map(|p| get_browser_ui_test_version(builder, p).is_some()) - .unwrap_or(false) - })) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocGUI { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); - - let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); - - if let Some(src) = builder.config.src.to_str() { - cmd.arg("--rust-src").arg(src); - } - - if let Some(out_dir) = out_dir.to_str() { - cmd.arg("--out-dir").arg(out_dir); - } - - if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { - cmd.arg("--initial-cargo").arg(initial_cargo); - } - - cmd.arg("--jobs").arg(builder.jobs().to_string()); - - cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) - .env("RUSTC", builder.rustc(self.compiler)); - - add_rustdoc_cargo_linker_args(&mut cmd, builder, self.compiler.host, LldThreads::No); - - for path in &builder.paths { - if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { - if !p.ends_with(".goml") { - eprintln!("A non-goml file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-gui tests"); - } - if let Some(name) = path.file_name().and_then(|f| f.to_str()) { - cmd.arg("--goml-file").arg(name); - } - } - } - - for test_arg in builder.config.test_args() { - cmd.arg("--test-arg").arg(test_arg); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } - - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - - let _time = helpers::timeit(builder); - let _guard = builder.msg_sysroot_tool( - Kind::Test, - self.compiler.stage, - "rustdoc-gui", - self.compiler.host, - self.target, - ); - try_run_tests(builder, &mut cmd, true); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Tidy; - -impl Step for Tidy { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Runs the `tidy` tool. - /// - /// This tool in `src/tools` checks up on various bits and pieces of style and - /// otherwise just implements a few lint-like checks that are specific to the - /// compiler itself. - /// - /// Once tidy passes, this step also runs `fmt --check` if tests are being run - /// for the `dev` or `nightly` channels. - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::Tidy); - cmd.arg(&builder.src); - cmd.arg(&builder.initial_cargo); - cmd.arg(&builder.out); - // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. - let jobs = builder.config.jobs.unwrap_or_else(|| { - 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }); - cmd.arg(jobs.to_string()); - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - if let Some(s) = builder.config.cmd.extra_checks() { - cmd.arg(format!("--extra-checks={s}")); - } - let mut args = std::env::args_os(); - if args.any(|arg| arg == OsStr::new("--")) { - cmd.arg("--"); - cmd.args(args); - } - - if builder.config.channel == "dev" || builder.config.channel == "nightly" { - builder.info("fmt check"); - if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); - eprintln!( - "\ -ERROR: no `rustfmt` binary found in {PATH} -INFO: `rust.channel` is currently set to \"{CHAN}\" -HELP: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file -HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", - PATH = inferred_rustfmt_dir.display(), - CHAN = builder.config.channel, - ); - crate::exit!(1); - } - let all = false; - crate::core::build_steps::format::format(builder, !builder.config.cmd.bless(), all, &[ - ]); - } - - builder.info("tidy check"); - cmd.delay_failure().run(builder); - - builder.info("x.py completions check"); - let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if builder.config.cmd.bless() { - builder.ensure(crate::core::build_steps::run::GenerateCompletions); - } else if get_completion(shells::Bash, &bash).is_some() - || get_completion(shells::Fish, &fish).is_some() - || get_completion(shells::PowerShell, &powershell).is_some() - || crate::flags::get_completion(shells::Zsh, &zsh).is_some() - { - eprintln!( - "x.py completions were changed; run `x.py run generate-completions` to update them" - ); - crate::exit!(1); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.doc_tests != DocTests::Only; - run.path("src/tools/tidy").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Tidy); - } -} - -fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { - builder.out.join(host).join("test") -} - -macro_rules! default_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); - }; -} - -macro_rules! default_test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, - compare_mode: $compare_mode:expr }) => { - test_with_compare_mode!($name { - path: $path, - mode: $mode, - suite: $suite, - default: true, - host: false, - compare_mode: $compare_mode - }); - }; -} - -macro_rules! host_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); - }; -} - -macro_rules! test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: None - }); - }; -} - -macro_rules! test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr, compare_mode: $compare_mode:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: Some($compare_mode) - }); - }; -} - -macro_rules! test_definitions { - ($name:ident { - path: $path:expr, - mode: $mode:expr, - suite: $suite:expr, - default: $default:expr, - host: $host:expr, - compare_mode: $compare_mode:expr - }) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = $host; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path($path) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - - run.builder.ensure($name { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: $mode, - suite: $suite, - path: $path, - compare_mode: $compare_mode, - }) - } - } - }; -} - -/// Declares an alias for running the [`Coverage`] tests in only one mode. -/// Adapted from [`test_definitions`]. -macro_rules! coverage_test_alias { - ($name:ident { - alias_and_mode: $alias_and_mode:expr, // &'static str - default: $default:expr, // bool - only_hosts: $only_hosts:expr $(,)? // bool - }) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl $name { - const MODE: &'static str = $alias_and_mode; - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = $only_hosts; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Register the mode name as a command-line alias. - // This allows `x test coverage-map` and `x test coverage-run`. - run.alias($alias_and_mode) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - - run.builder.ensure($name { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - Coverage::run_coverage_tests(builder, self.compiler, self.target, Self::MODE); - } - } - }; -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct RunMakeSupport { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMakeSupport { - type Output = PathBuf; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - } - - /// Builds run-make-support and returns the path to the resulting rlib. - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.target, - Kind::Build, - "src/tools/run-make-support", - SourceType::InTree, - &[], - ); - - cargo.into_cmd().run(builder); - - let lib_name = "librun_make_support.rlib"; - let lib = builder.tools_dir(self.compiler).join(lib_name); - - let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(lib_name); - builder.copy_link(&cargo_out, &lib); - lib - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateRunMakeSupport { - host: TargetSelection, -} - -impl Step for CrateRunMakeSupport { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/run-make-support") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrateRunMakeSupport { host: run.target }); - } - - /// Runs `cargo test` for run-make-support. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(0, host); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - host, - Kind::Test, - "src/tools/run-make-support", - SourceType::InTree, - &[], - ); - cargo.allow_features("test"); - run_cargo_test( - cargo, - &[], - &[], - "run-make-support", - "run-make-support self test", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateBuildHelper { - host: TargetSelection, -} - -impl Step for CrateBuildHelper { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/build_helper") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrateBuildHelper { host: run.target }); - } - - /// Runs `cargo test` for build_helper. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(0, host); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - host, - Kind::Test, - "src/build_helper", - SourceType::InTree, - &[], - ); - cargo.allow_features("test"); - run_cargo_test( - cargo, - &[], - &[], - "build_helper", - "build_helper self test", - compiler, - host, - builder, - ); - } -} - -default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); - -default_test!(Crashes { path: "tests/crashes", mode: "crashes", suite: "crashes" }); - -default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); - -default_test!(CodegenUnits { - path: "tests/codegen-units", - mode: "codegen-units", - suite: "codegen-units" -}); - -default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); - -default_test_with_compare_mode!(Debuginfo { - path: "tests/debuginfo", - mode: "debuginfo", - suite: "debuginfo", - compare_mode: "split-dwarf" -}); - -host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); - -host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); -host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); - -host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); - -host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); - -/// Special-handling is needed for `run-make`, so don't use `default_test` for defining `RunMake` -/// tests. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RunMake { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMake { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/run-make") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - run.builder.ensure(RunMake { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "run-make", - suite: "run-make", - path: "tests/run-make", - compare_mode: None, - }); - } -} - -default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); - -/// Coverage tests are a bit more complicated than other test suites, because -/// we want to run the same set of test files in multiple different modes, -/// in a way that's convenient and flexible when invoked manually. -/// -/// This combined step runs the specified tests (or all of `tests/coverage`) -/// in both "coverage-map" and "coverage-run" modes. -/// -/// Used by: -/// - `x test coverage` -/// - `x test tests/coverage` -/// - `x test tests/coverage/trivial.rs` (etc) -/// -/// (Each individual mode also has its own step that will run the tests in -/// just that mode.) -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Coverage { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Coverage { - const PATH: &'static str = "tests/coverage"; - const SUITE: &'static str = "coverage"; - - /// Runs the coverage test suite (or a user-specified subset) in one mode. - /// - /// This same function is used by the multi-mode step ([`Coverage`]) and by - /// the single-mode steps ([`CoverageMap`] and [`CoverageRun`]), to help - /// ensure that they all behave consistently with each other, regardless of - /// how the coverage tests have been invoked. - fn run_coverage_tests( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - mode: &'static str, - ) { - // Like many other test steps, we delegate to a `Compiletest` step to - // actually run the tests. (See `test_definitions!`.) - builder.ensure(Compiletest { - compiler, - target, - mode, - suite: Self::SUITE, - path: Self::PATH, - compare_mode: None, - }); - } -} - -impl Step for Coverage { - type Output = (); - /// We rely on the individual CoverageMap/CoverageRun steps to run themselves. - const DEFAULT: bool = false; - /// When manually invoked, try to run as much as possible. - /// Compiletest will automatically skip the "coverage-run" tests if necessary. - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Take responsibility for command-line paths within `tests/coverage`. - run.suite_path(Self::PATH) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - - run.builder.ensure(Coverage { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - // Run the specified coverage tests (possibly all of them) in both modes. - Self::run_coverage_tests(builder, self.compiler, self.target, CoverageMap::MODE); - Self::run_coverage_tests(builder, self.compiler, self.target, CoverageRun::MODE); - } -} - -// Runs `tests/coverage` in "coverage-map" mode only. -// Used by `x test` and `x test coverage-map`. -coverage_test_alias!(CoverageMap { - alias_and_mode: "coverage-map", - default: true, - only_hosts: false, -}); -// Runs `tests/coverage` in "coverage-run" mode only. -// Used by `x test` and `x test coverage-run`. -coverage_test_alias!(CoverageRun { - alias_and_mode: "coverage-run", - default: true, - // Compiletest knows how to automatically skip these tests when cross-compiling, - // but skipping the whole step here makes it clearer that they haven't run at all. - only_hosts: true, -}); - -host_test!(CoverageRunRustdoc { - path: "tests/coverage-run-rustdoc", - mode: "coverage-run", - suite: "coverage-run-rustdoc" -}); - -// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct MirOpt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for MirOpt { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/mir-opt") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(MirOpt { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let run = |target| { - builder.ensure(Compiletest { - compiler: self.compiler, - target, - mode: "mir-opt", - suite: "mir-opt", - path: "tests/mir-opt", - compare_mode: None, - }) - }; - - run(self.target); - - // Run more targets with `--bless`. But we always run the host target first, since some - // tests use very specific `only` clauses that are not covered by the target set below. - if builder.config.cmd.bless() { - // All that we really need to do is cover all combinations of 32/64-bit and unwind/abort, - // but while we're at it we might as well flex our cross-compilation support. This - // selection covers all our tier 1 operating systems and architectures using only tier - // 1 targets. - - for target in ["aarch64-unknown-linux-gnu", "i686-pc-windows-msvc"] { - run(TargetSelection::from_user(target)); - } - - for target in ["x86_64-apple-darwin", "i686-unknown-linux-musl"] { - let target = TargetSelection::from_user(target); - let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { - compiler: self.compiler, - base: target, - }); - run(panic_abort_target); - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct Compiletest { - compiler: Compiler, - target: TargetSelection, - mode: &'static str, - suite: &'static str, - path: &'static str, - compare_mode: Option<&'static str>, -} - -impl Step for Compiletest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Executes the `compiletest` tool to run a suite of tests. - /// - /// Compiles all tests with `compiler` for `target` with the specified - /// compiletest `mode` and `suite` arguments. For example `mode` can be - /// "run-pass" or `suite` can be something like `debuginfo`. - fn run(self, builder: &Builder<'_>) { - if builder.doc_tests == DocTests::Only { - return; - } - - if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { - eprintln!("\ -ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail -HELP: to test the compiler, use `--stage 1` instead -HELP: to test the standard library, use `--stage 0 library/std` instead -NOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." - ); - crate::exit!(1); - } - - let mut compiler = self.compiler; - let target = self.target; - let mode = self.mode; - let suite = self.suite; - - // Path for test suite - let suite_path = self.path; - - // Skip codegen tests if they aren't enabled in configuration. - if !builder.config.codegen_tests && suite == "codegen" { - return; - } - - // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most - // part test the *API* of the compiler, not how it compiles a given file. As a result, we - // can run them against the stage 1 sources as long as we build them with the stage 0 - // bootstrap compiler. - // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the - // running compiler in stage 2 when plugins run. - let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { - // At stage 0 (stage - 1) we are using the beta compiler. Using `self.target` can lead finding - // an incorrect compiler path on cross-targets, as the stage 0 beta compiler is always equal - // to `build.build` in the configuration. - let build = builder.build.build; - - compiler = builder.compiler(compiler.stage - 1, build); - format!("stage{}-{}", compiler.stage + 1, build) - } else { - format!("stage{}-{}", compiler.stage, target) - }; - - if suite.ends_with("fulldeps") { - builder.ensure(compile::Rustc::new(compiler, target)); - } - - if suite == "debuginfo" { - builder.ensure(dist::DebuggerScripts { - sysroot: builder.sysroot(compiler).to_path_buf(), - host: target, - }); - } - - // Also provide `rust_test_helpers` for the host. - builder.ensure(TestHelpers { target: compiler.host }); - - // ensure that `libproc_macro` is available on the host. - if suite == "mir-opt" { - builder.ensure(compile::Std::new_for_mir_opt_tests(compiler, compiler.host)); - } else { - builder.ensure(compile::Std::new(compiler, compiler.host)); - } - - // As well as the target - if suite != "mir-opt" { - builder.ensure(TestHelpers { target }); - } - - let mut cmd = builder.tool_cmd(Tool::Compiletest); - - if suite == "mir-opt" { - builder.ensure(compile::Std::new_for_mir_opt_tests(compiler, target)); - } else { - builder.ensure(compile::Std::new(compiler, target)); - } - - builder.ensure(RemoteCopyLibs { compiler, target }); - - // compiletest currently has... a lot of arguments, so let's just pass all - // of them! - - cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); - cmd.arg("--run-lib-path").arg(builder.sysroot_target_libdir(compiler, target)); - cmd.arg("--rustc-path").arg(builder.rustc(compiler)); - - // Minicore auxiliary lib for `no_core` tests that need `core` stubs in cross-compilation - // scenarios. - cmd.arg("--minicore-path") - .arg(builder.src.join("tests").join("auxiliary").join("minicore.rs")); - - let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); - - if mode == "run-make" { - let cargo_path = if builder.top_stage == 0 { - // If we're using `--stage 0`, we should provide the bootstrap cargo. - builder.initial_cargo.clone() - } else { - // We need to properly build cargo using the suitable stage compiler. - - let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(|| - // HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if - // you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built - // and produce a cargo built with stage 2 rustc. To fix this, we need to chop off - // the compiler stage by 1 to align with expected `./x test run-make --stage N` - // behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri - // which does a similar hack. - builder.compiler(builder.top_stage - 1, compiler.host)); - - builder.ensure(tool::Cargo { compiler, target: compiler.host }) - }; - - cmd.arg("--cargo-path").arg(cargo_path); - } - - // Avoid depending on rustdoc when we don't need it. - if mode == "rustdoc" - || mode == "run-make" - || (mode == "ui" && is_rustdoc) - || mode == "js-doc-test" - || mode == "rustdoc-json" - || suite == "coverage-run-rustdoc" - { - cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); - } - - if mode == "rustdoc-json" { - // Use the beta compiler for jsondocck - let json_compiler = compiler.with_stage(0); - cmd.arg("--jsondocck-path") - .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); - cmd.arg("--jsondoclint-path") - .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); - } - - if matches!(mode, "coverage-map" | "coverage-run") { - let coverage_dump = builder.tool_exe(Tool::CoverageDump); - cmd.arg("--coverage-dump-path").arg(coverage_dump); - } - - cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); - cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); - - // When top stage is 0, that means that we're testing an externally provided compiler. - // In that case we need to use its specific sysroot for tests to pass. - let sysroot = if builder.top_stage == 0 { - builder.initial_sysroot.clone() - } else { - builder.sysroot(compiler).to_path_buf() - }; - cmd.arg("--sysroot-base").arg(sysroot); - cmd.arg("--stage-id").arg(stage_id); - cmd.arg("--suite").arg(suite); - cmd.arg("--mode").arg(mode); - cmd.arg("--target").arg(target.rustc_target_arg()); - cmd.arg("--host").arg(&*compiler.host.triple); - cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); - - if builder.build.config.llvm_enzyme { - cmd.arg("--has-enzyme"); - } - - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - - if builder.config.cmd.force_rerun() { - cmd.arg("--force-rerun"); - } - - let compare_mode = - builder.config.cmd.compare_mode().or_else(|| { - if builder.config.test_compare_mode { self.compare_mode } else { None } - }); - - if let Some(ref pass) = builder.config.cmd.pass() { - cmd.arg("--pass"); - cmd.arg(pass); - } - - if let Some(ref run) = builder.config.cmd.run() { - cmd.arg("--run"); - cmd.arg(run); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } else if mode == "js-doc-test" { - panic!("need nodejs to run js-doc-test suite"); - } - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - if builder.config.rust_optimize_tests { - cmd.arg("--optimize-tests"); - } - if builder.config.rust_randomize_layout { - cmd.arg("--rust-randomized-layout"); - } - if builder.config.cmd.only_modified() { - cmd.arg("--only-modified"); - } - if let Some(compiletest_diff_tool) = &builder.config.compiletest_diff_tool { - cmd.arg("--compiletest-diff-tool").arg(compiletest_diff_tool); - } - - let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; - flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); - flags.extend(builder.config.cmd.compiletest_rustc_args().iter().map(|s| s.to_string())); - - if suite != "mir-opt" { - if let Some(linker) = builder.linker(target) { - cmd.arg("--target-linker").arg(linker); - } - if let Some(linker) = builder.linker(compiler.host) { - cmd.arg("--host-linker").arg(linker); - } - } - - let mut hostflags = flags.clone(); - hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); - hostflags.extend(linker_flags(builder, compiler.host, LldThreads::No)); - for flag in hostflags { - cmd.arg("--host-rustcflags").arg(flag); - } - - let mut targetflags = flags; - targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); - targetflags.extend(linker_flags(builder, compiler.host, LldThreads::No)); - for flag in targetflags { - cmd.arg("--target-rustcflags").arg(flag); - } - - cmd.arg("--python").arg(builder.python()); - - if let Some(ref gdb) = builder.config.gdb { - cmd.arg("--gdb").arg(gdb); - } - - let lldb_exe = builder.config.lldb.clone().unwrap_or_else(|| PathBuf::from("lldb")); - let lldb_version = command(&lldb_exe) - .allow_failure() - .arg("--version") - .run_capture(builder) - .stdout_if_ok() - .and_then(|v| if v.trim().is_empty() { None } else { Some(v) }); - if let Some(ref vers) = lldb_version { - cmd.arg("--lldb-version").arg(vers); - let lldb_python_dir = command(&lldb_exe) - .allow_failure() - .arg("-P") - .run_capture_stdout(builder) - .stdout_if_ok() - .map(|p| p.lines().next().expect("lldb Python dir not found").to_string()); - if let Some(ref dir) = lldb_python_dir { - cmd.arg("--lldb-python-dir").arg(dir); - } - } - - if helpers::forcing_clang_based_tests() { - let clang_exe = builder.llvm_out(target).join("bin").join("clang"); - cmd.arg("--run-clang-based-tests-with").arg(clang_exe); - } - - for exclude in &builder.config.skip { - cmd.arg("--skip"); - cmd.arg(exclude); - } - - // Get paths from cmd args - let paths = match &builder.config.cmd { - Subcommand::Test { .. } => &builder.config.paths[..], - _ => &[], - }; - - // Get test-args by striping suite path - let mut test_args: Vec<&str> = paths - .iter() - .filter_map(|p| helpers::is_valid_test_suite_arg(p, suite_path, builder)) - .collect(); - - test_args.append(&mut builder.config.test_args()); - - // On Windows, replace forward slashes in test-args by backslashes - // so the correct filters are passed to libtest - if cfg!(windows) { - let test_args_win: Vec = - test_args.iter().map(|s| s.replace('/', "\\")).collect(); - cmd.args(&test_args_win); - } else { - cmd.args(&test_args); - } - - if builder.is_verbose() { - cmd.arg("--verbose"); - } - - cmd.arg("--json"); - - if builder.config.rustc_debug_assertions { - cmd.arg("--with-rustc-debug-assertions"); - } - - if builder.config.std_debug_assertions { - cmd.arg("--with-std-debug-assertions"); - } - - let mut llvm_components_passed = false; - let mut copts_passed = false; - if builder.config.llvm_enabled(compiler.host) { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_version = - command(&llvm_config).arg("--version").run_capture_stdout(builder).stdout(); - let llvm_components = - command(&llvm_config).arg("--components").run_capture_stdout(builder).stdout(); - // Remove trailing newline from llvm-config output. - cmd.arg("--llvm-version") - .arg(llvm_version.trim()) - .arg("--llvm-components") - .arg(llvm_components.trim()); - llvm_components_passed = true; - } - if !builder.is_rust_llvm(target) { - // FIXME: missing Rust patches is not the same as being system llvm; we should rename the flag at some point. - // Inspecting the tests with `// no-system-llvm` in src/test *looks* like this is doing the right thing, though. - cmd.arg("--system-llvm"); - } - - // Tests that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate compilations. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if !builder.config.dry_run() && suite.ends_with("fulldeps") { - let llvm_libdir = - command(&llvm_config).arg("--libdir").run_capture_stdout(builder).stdout(); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); - } - - if !builder.config.dry_run() && matches!(mode, "run-make" | "coverage-run") { - // The llvm/bin directory contains many useful cross-platform - // tools. Pass the path to run-make tests so they can use them. - // (The coverage-run tests also need these tools to process - // coverage reports.) - let llvm_bin_path = llvm_config - .parent() - .expect("Expected llvm-config to be contained in directory"); - assert!(llvm_bin_path.is_dir()); - cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); - } - - if !builder.config.dry_run() && mode == "run-make" { - // If LLD is available, add it to the PATH - if builder.config.lld_enabled { - let lld_install_root = - builder.ensure(llvm::Lld { target: builder.config.build }); - - let lld_bin_path = lld_install_root.join("bin"); - - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths( - std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), - ) - .expect("Could not add LLD bin path to PATH"); - cmd.env("PATH", new_path); - } - } - } - - // Only pass correct values for these flags for the `run-make` suite as it - // requires that a C++ compiler was configured which isn't always the case. - if !builder.config.dry_run() && mode == "run-make" { - cmd.arg("--cc") - .arg(builder.cc(target)) - .arg("--cxx") - .arg(builder.cxx(target).unwrap()) - .arg("--cflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) - .arg("--cxxflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); - copts_passed = true; - if let Some(ar) = builder.ar(target) { - cmd.arg("--ar").arg(ar); - } - } - - if !llvm_components_passed { - cmd.arg("--llvm-components").arg(""); - } - if !copts_passed { - cmd.arg("--cc") - .arg("") - .arg("--cxx") - .arg("") - .arg("--cflags") - .arg("") - .arg("--cxxflags") - .arg(""); - } - - if builder.remote_tested(target) { - cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); - } else if let Some(tool) = builder.runner(target) { - cmd.arg("--runner").arg(tool); - } - - if suite != "mir-opt" { - // Running a C compiler on MSVC requires a few env vars to be set, to be - // sure to set them here. - // - // Note that if we encounter `PATH` we make sure to append to our own `PATH` - // rather than stomp over it. - if !builder.config.dry_run() && target.is_msvc() { - for (k, v) in builder.cc.borrow()[&target].env() { - if k != "PATH" { - cmd.env(k, v); - } - } - } - } - - // Special setup to enable running with sanitizers on MSVC. - if !builder.config.dry_run() - && target.contains("msvc") - && builder.config.sanitizers_enabled(target) - { - // Ignore interception failures: not all dlls in the process will have been built with - // address sanitizer enabled (e.g., ntdll.dll). - cmd.env("ASAN_WIN_CONTINUE_ON_INTERCEPTION_FAILURE", "1"); - // Add the address sanitizer runtime to the PATH - it is located next to cl.exe. - let asan_runtime_path = - builder.cc.borrow()[&target].path().parent().unwrap().to_path_buf(); - let old_path = cmd - .get_envs() - .find_map(|(k, v)| (k == "PATH").then_some(v)) - .flatten() - .map_or_else(|| env::var_os("PATH").unwrap_or_default(), |v| v.to_owned()); - let new_path = env::join_paths( - env::split_paths(&old_path).chain(std::iter::once(asan_runtime_path)), - ) - .expect("Could not add ASAN runtime path to PATH"); - cmd.env("PATH", new_path); - } - - // Some UI tests trigger behavior in rustc where it reads $CARGO and changes behavior if it exists. - // To make the tests work that rely on it not being set, make sure it is not set. - cmd.env_remove("CARGO"); - - cmd.env("RUSTC_BOOTSTRAP", "1"); - // Override the rustc version used in symbol hashes to reduce the amount of normalization - // needed when diffing test output. - cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); - cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - builder.add_rust_test_threads(&mut cmd); - - if builder.config.sanitizers_enabled(target) { - cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); - } - - if builder.config.profiler_enabled(target) { - cmd.arg("--profiler-runtime"); - } - - cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); - - cmd.arg("--adb-path").arg("adb"); - cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); - if target.contains("android") && !builder.config.dry_run() { - // Assume that cc for this target comes from the android sysroot - cmd.arg("--android-cross-path") - .arg(builder.cc(target).parent().unwrap().parent().unwrap()); - } else { - cmd.arg("--android-cross-path").arg(""); - } - - if builder.config.cmd.rustfix_coverage() { - cmd.arg("--rustfix-coverage"); - } - - cmd.arg("--channel").arg(&builder.config.channel); - - if !builder.config.omit_git_hash { - cmd.arg("--git-hash"); - } - - let git_config = builder.config.git_config(); - cmd.arg("--git-repository").arg(git_config.git_repository); - cmd.arg("--nightly-branch").arg(git_config.nightly_branch); - cmd.arg("--git-merge-commit-email").arg(git_config.git_merge_commit_email); - cmd.force_coloring_in_ci(); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: None, - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - let _group = builder.msg( - Kind::Test, - compiler.stage, - format!("compiletest suite={suite} mode={mode}"), - compiler.host, - target, - ); - try_run_tests(builder, &mut cmd, false); - - if let Some(compare_mode) = compare_mode { - cmd.arg("--compare-mode").arg(compare_mode); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: Some(compare_mode.into()), - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - builder.info(&format!( - "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", - suite, mode, compare_mode, &compiler.host, target - )); - let _time = helpers::timeit(builder); - try_run_tests(builder, &mut cmd, false); - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct BookTest { - compiler: Compiler, - path: PathBuf, - name: &'static str, - is_ext_doc: bool, -} - -impl Step for BookTest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Runs the documentation tests for a book in `src/doc`. - /// - /// This uses the `rustdoc` that sits next to `compiler`. - fn run(self, builder: &Builder<'_>) { - // External docs are different from local because: - // - Some books need pre-processing by mdbook before being tested. - // - They need to save their state to toolstate. - // - They are only tested on the "checktools" builders. - // - // The local docs are tested by default, and we don't want to pay the - // cost of building mdbook, so they use `rustdoc --test` directly. - // Also, the unstable book is special because SUMMARY.md is generated, - // so it is easier to just run `rustdoc` on its files. - if self.is_ext_doc { - self.run_ext_doc(builder); - } else { - self.run_local_doc(builder); - } - } -} - -impl BookTest { - /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) - /// which in turn runs `rustdoc --test` on each file in the book. - fn run_ext_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - builder.ensure(compile::Std::new(compiler, compiler.host)); - - // mdbook just executes a binary named "rustdoc", so we need to update - // PATH so that it points to our rustdoc. - let mut rustdoc_path = builder.rustdoc(compiler); - rustdoc_path.pop(); - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) - .expect("could not add rustdoc to PATH"); - - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let path = builder.src.join(&self.path); - // Books often have feature-gated example text. - rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); - rustbook_cmd.env("PATH", new_path).arg("test").arg(path); - builder.add_rust_test_threads(&mut rustbook_cmd); - let _guard = builder.msg( - Kind::Test, - compiler.stage, - format_args!("mdbook {}", self.path.display()), - compiler.host, - compiler.host, - ); - let _time = helpers::timeit(builder); - let toolstate = if rustbook_cmd.delay_failure().run(builder) { - ToolState::TestPass - } else { - ToolState::TestFail - }; - builder.save_toolstate(self.name, toolstate); - } - - /// This runs `rustdoc --test` on all `.md` files in the path. - fn run_local_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let host = self.compiler.host; - - builder.ensure(compile::Std::new(compiler, host)); - - let _guard = - builder.msg(Kind::Test, compiler.stage, format!("book {}", self.name), host, host); - - // Do a breadth-first traversal of the `src/doc` directory and just run - // tests for all files that end in `*.md` - let mut stack = vec![builder.src.join(self.path)]; - let _time = helpers::timeit(builder); - let mut files = Vec::new(); - while let Some(p) = stack.pop() { - if p.is_dir() { - stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); - continue; - } - - if p.extension().and_then(|s| s.to_str()) != Some("md") { - continue; - } - - files.push(p); - } - - files.sort(); - - for file in files { - markdown_test(builder, compiler, &file); - } - } -} - -macro_rules! test_book { - ($( - $name:ident, $path:expr, $book_name:expr, - default=$default:expr - $(,submodules = $submodules:expr)? - ; - )+) => { - $( - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) { - $( - for submodule in $submodules { - builder.require_submodule(submodule, None); - } - )* - builder.ensure(BookTest { - compiler: self.compiler, - path: PathBuf::from($path), - name: $book_name, - is_ext_doc: !$default, - }); - } - } - )+ - } -} - -test_book!( - Nomicon, "src/doc/nomicon", "nomicon", default=false, submodules=["src/doc/nomicon"]; - Reference, "src/doc/reference", "reference", default=false, submodules=["src/doc/reference"]; - RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; - RustcBook, "src/doc/rustc", "rustc", default=true; - RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false, submodules=["src/doc/rust-by-example"]; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false, submodules=["src/doc/embedded-book"]; - TheBook, "src/doc/book", "book", default=false, submodules=["src/doc/book"]; - UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; - EditionGuide, "src/doc/edition-guide", "edition-guide", default=false, submodules=["src/doc/edition-guide"]; -); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ErrorIndex { - compiler: Compiler, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // error_index_generator depends on librustdoc. Use the compiler that - // is normally used to build rustdoc for other tests (like compiletest - // tests in tests/rustdoc) so that it shares the same artifacts. - let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - /// Runs the error index generator tool to execute the tests located in the error - /// index. - /// - /// The `error_index_generator` tool lives in `src/tools` and is used to - /// generate a markdown file from the error indexes of the code base which is - /// then passed to `rustdoc --test`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - let output = dir.join("error-index.md"); - - let mut tool = tool::ErrorIndex::command(builder); - tool.arg("markdown").arg(&output); - - let guard = - builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); - let _time = helpers::timeit(builder); - tool.run_capture(builder); - drop(guard); - // The tests themselves need to link to std, so make sure it is - // available. - builder.ensure(compile::Std::new(compiler, compiler.host)); - markdown_test(builder, compiler, &output); - } -} - -fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { - if let Ok(contents) = fs::read_to_string(markdown) { - if !contents.contains("```") { - return true; - } - } - - builder.verbose(|| println!("doc tests for: {}", markdown.display())); - let mut cmd = builder.rustdoc_cmd(compiler); - builder.add_rust_test_threads(&mut cmd); - // allow for unstable options such as new editions - cmd.arg("-Z"); - cmd.arg("unstable-options"); - cmd.arg("--test"); - cmd.arg(markdown); - cmd.env("RUSTC_BOOTSTRAP", "1"); - - let test_args = builder.config.test_args().join(" "); - cmd.arg("--test-args").arg(test_args); - - cmd = cmd.delay_failure(); - if !builder.config.verbose_tests { - cmd.run_capture(builder).is_success() - } else { - cmd.run(builder) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustcGuide; - -impl Step for RustcGuide { - type Output = (); - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/doc/rustc-dev-guide") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcGuide); - } - - fn run(self, builder: &Builder<'_>) { - let relative_path = "src/doc/rustc-dev-guide"; - builder.require_submodule(relative_path, None); - - let src = builder.src.join(relative_path); - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook).delay_failure(); - rustbook_cmd.arg("linkcheck").arg(&src); - let toolstate = - if rustbook_cmd.run(builder) { ToolState::TestPass } else { ToolState::TestFail }; - builder.save_toolstate("rustc-dev-guide", toolstate); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateLibrustc { - compiler: Compiler, - target: TargetSelection, - crates: Vec, -} - -impl Step for CrateLibrustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run.make_run_crates(Alias::Compiler); - - builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - builder.ensure(Crate { - compiler: self.compiler, - target: self.target, - mode: Mode::Rustc, - crates: self.crates, - }); - } -} - -/// Given a `cargo test` subcommand, add the appropriate flags and run it. -/// -/// Returns whether the test succeeded. -#[allow(clippy::too_many_arguments)] // FIXME: reduce the number of args and remove this. -fn run_cargo_test<'a>( - cargo: impl Into, - libtest_args: &[&str], - crates: &[String], - primary_crate: &str, - description: impl Into>, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> bool { - let mut cargo = - prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); - let _time = helpers::timeit(builder); - let _group = description.into().and_then(|what| { - builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) - }); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: crates.iter().map(|c| c.to_string()).collect(), - target: target.triple.to_string(), - host: compiler.host.triple.to_string(), - stage: compiler.stage, - }, - builder, - ); - add_flags_and_try_run_tests(builder, &mut cargo) -} - -/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. -fn prepare_cargo_test( - cargo: impl Into, - libtest_args: &[&str], - crates: &[String], - primary_crate: &str, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> BootstrapCommand { - let mut cargo = cargo.into(); - - // Propegate `--bless` if it has not already been set/unset - // Any tools that want to use this should bless if `RUSTC_BLESS` is set to - // anything other than `0`. - if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { - cargo.env("RUSTC_BLESS", "Gesundheit"); - } - - // Pass in some standard flags then iterate over the graph we've discovered - // in `cargo metadata` with the maps above and figure out what `-p` - // arguments need to get passed. - if builder.kind == Kind::Test && !builder.fail_fast { - cargo.arg("--no-fail-fast"); - } - match builder.doc_tests { - DocTests::Only => { - cargo.arg("--doc"); - } - DocTests::No => { - let krate = &builder - .crates - .get(primary_crate) - .unwrap_or_else(|| panic!("missing crate {primary_crate}")); - if krate.has_lib { - cargo.arg("--lib"); - } - cargo.args(["--bins", "--examples", "--tests", "--benches"]); - } - DocTests::Yes => {} - } - - for krate in crates { - cargo.arg("-p").arg(krate); - } - - cargo.arg("--").args(builder.config.test_args()).args(libtest_args); - if !builder.config.verbose_tests { - cargo.arg("--quiet"); - } - - // The tests are going to run with the *target* libraries, so we need to - // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. - // - // Note that to run the compiler we need to run with the *host* libraries, - // but our wrapper scripts arrange for that to be the case anyway. - // - // We skip everything on Miri as then this overwrites the libdir set up - // by `Cargo::new` and that actually makes things go wrong. - if builder.kind != Kind::Miri { - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*builder.sysroot_target_libdir(compiler, target))); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - } - - if builder.remote_tested(target) { - cargo.env( - format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), - format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), - ); - } else if let Some(tool) = builder.runner(target) { - cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), tool); - } - - cargo -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Crate { - pub compiler: Compiler, - pub target: TargetSelection, - pub mode: Mode, - pub crates: Vec, -} - -impl Step for Crate { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run - .paths - .iter() - .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) - .collect(); - - builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); - } - - /// Runs all unit tests plus documentation tests for a given crate defined - /// by a `Cargo.toml` (single manifest) - /// - /// This is what runs tests for crates like the standard library, compiler, etc. - /// It essentially is the driver for running `cargo test`. - /// - /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` - /// arguments, and those arguments are discovered from `cargo metadata`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let mode = self.mode; - - // Prepare sysroot - // See [field@compile::Std::force_recompile]. - builder.ensure(compile::Std::force_recompile(compiler, compiler.host)); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let mut cargo = if builder.kind == Kind::Miri { - if builder.top_stage == 0 { - eprintln!("ERROR: `x.py miri` requires stage 1 or higher"); - std::process::exit(1); - } - - // Build `cargo miri test` command - // (Implicitly prepares target sysroot) - let mut cargo = builder::Cargo::new( - builder, - compiler, - mode, - SourceType::InTree, - target, - Kind::MiriTest, - ); - // This hack helps bootstrap run standard library tests in Miri. The issue is as - // follows: when running `cargo miri test` on libcore, cargo builds a local copy of core - // and makes it a dependency of the integration test crate. This copy duplicates all the - // lang items, so the build fails. (Regular testing avoids this because the sysroot is a - // literal copy of what `cargo build` produces, but since Miri builds its own sysroot - // this does not work for us.) So we need to make it so that the locally built libcore - // contains all the items from `core`, but does not re-define them -- we want to replace - // the entire crate but a re-export of the sysroot crate. We do this by swapping out the - // source file: if `MIRI_REPLACE_LIBRS_IF_NOT_TEST` is set and we are building a - // `lib.rs` file, and a `lib.miri.rs` file exists in the same folder, we build that - // instead. But crucially we only do that for the library, not the test builds. - cargo.env("MIRI_REPLACE_LIBRS_IF_NOT_TEST", "1"); - cargo - } else { - // Also prepare a sysroot for the target. - if builder.config.build != target { - builder.ensure(compile::Std::force_recompile(compiler, target)); - builder.ensure(RemoteCopyLibs { compiler, target }); - } - - // Build `cargo test` command - builder::Cargo::new(builder, compiler, mode, SourceType::InTree, target, builder.kind) - }; - - match mode { - Mode::Std => { - if builder.kind == Kind::Miri { - // We can't use `std_cargo` as that uses `optimized-compiler-builtins` which - // needs host tools for the given target. This is similar to what `compile::Std` - // does when `is_for_mir_opt_tests` is true. There's probably a chance for - // de-duplication here... `std_cargo` should support a mode that avoids needing - // host tools. - cargo - .arg("--manifest-path") - .arg(builder.src.join("library/sysroot/Cargo.toml")); - } else { - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, - // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. - // Override it. - if builder.download_rustc() && compiler.stage > 0 { - let sysroot = builder - .out - .join(compiler.host) - .join(format!("stage{}-test-sysroot", compiler.stage)); - cargo.env("RUSTC_SYSROOT", sysroot); - } - } - } - Mode::Rustc => { - compile::rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - } - _ => panic!("can only test libraries"), - }; - - run_cargo_test( - cargo, - &[], - &self.crates, - &self.crates[0], - &*crate_description(&self.crates), - compiler, - target, - builder, - ); - } -} - -/// Rustdoc is special in various ways, which is why this step is different from `Crate`. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdoc { - host: TargetSelection, -} - -impl Step for CrateRustdoc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["src/librustdoc", "src/tools/rustdoc"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdoc { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - let compiler = if builder.download_rustc() { - builder.compiler(builder.top_stage, target) - } else { - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - builder.compiler_for(builder.top_stage, target, target) - }; - // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when - // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from - // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this - // explicitly to make sure it ends up in the stage2 sysroot. - builder.ensure(compile::Std::new(compiler, target)); - builder.ensure(compile::Rustc::new(compiler, target)); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind, - "src/tools/rustdoc", - SourceType::InTree, - &[], - ); - if self.host.contains("musl") { - cargo.arg("'-Ctarget-feature=-crt-static'"); - } - - // This is needed for running doctests on librustdoc. This is a bit of - // an unfortunate interaction with how bootstrap works and how cargo - // sets up the dylib path, and the fact that the doctest (in - // html/markdown.rs) links to rustc-private libs. For stage1, the - // compiler host dylibs (in stage1/lib) are not the same as the target - // dylibs (in stage1/lib/rustlib/...). This is different from a normal - // rust distribution where they are the same. - // - // On the cargo side, normal tests use `target_process` which handles - // setting up the dylib for a *target* (stage1/lib/rustlib/... in this - // case). However, for doctests it uses `rustdoc_process` which only - // sets up the dylib path for the *host* (stage1/lib), which is the - // wrong directory. - // - // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. - // - // It should be considered to just stop running doctests on - // librustdoc. There is only one test, and it doesn't look too - // important. There might be other ways to avoid this, but it seems - // pretty convoluted. - // - // See also https://github.com/rust-lang/rust/issues/13983 where the - // host vs target dylibs for rustdoc are consistently tricky to deal - // with. - // - // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. - let libdir = if builder.download_rustc() { - builder.rustc_libdir(compiler) - } else { - builder.sysroot_target_libdir(compiler, target).to_path_buf() - }; - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*libdir)); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - run_cargo_test( - cargo, - &[], - &["rustdoc:0.0.0".to_string()], - "rustdoc", - "rustdoc", - compiler, - target, - builder, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdocJsonTypes { - host: TargetSelection, -} - -impl Step for CrateRustdocJsonTypes { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/rustdoc-json-types") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdocJsonTypes { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - let compiler = builder.compiler_for(builder.top_stage, target, target); - builder.ensure(compile::Rustc::new(compiler, target)); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind, - "src/rustdoc-json-types", - SourceType::InTree, - &[], - ); - - // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. - let libtest_args = if self.host.contains("musl") { - ["'-Ctarget-feature=-crt-static'"].as_slice() - } else { - &[] - }; - - run_cargo_test( - cargo, - libtest_args, - &["rustdoc-json-types".to_string()], - "rustdoc-json-types", - "rustdoc-json-types", - compiler, - target, - builder, - ); - } -} - -/// Some test suites are run inside emulators or on remote devices, and most -/// of our test binaries are linked dynamically which means we need to ship -/// the standard library and such to the emulator ahead of time. This step -/// represents this and is a dependency of all test suites. +/// Coverage tests are a bit more complicated than other test suites, because +/// we want to run the same set of test files in multiple different modes, +/// in a way that's convenient and flexible when invoked manually. /// -/// Most of the time this is a no-op. For some steps such as shipping data to -/// QEMU we have to build our own tools so we've got conditional dependencies -/// on those programs as well. Note that the remote test client is built for -/// the build target (us) and the server is built for the target. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RemoteCopyLibs { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for RemoteCopyLibs { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - if !builder.remote_tested(target) { - return; - } - - builder.ensure(compile::Std::new(compiler, target)); - - builder.info(&format!("REMOTE copy libs to emulator ({target})")); - - let server = builder.ensure(tool::RemoteTestServer { compiler, target }); - - // Spawn the emulator and wait for it to come online - let tool = builder.tool_exe(Tool::RemoteTestClient); - let mut cmd = command(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); - if let Some(rootfs) = builder.qemu_rootfs(target) { - cmd.arg(rootfs); - } - cmd.run(builder); - - // Push all our dylibs to the emulator - for f in t!(builder.sysroot_target_libdir(compiler, target).read_dir()) { - let f = t!(f); - if helpers::is_dylib(&f.path()) { - command(&tool).arg("push").arg(f.path()).run(builder); - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Distcheck; - -impl Step for Distcheck { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("distcheck") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Distcheck); - } - - /// Runs "distcheck", a 'make check' from a tarball - fn run(self, builder: &Builder<'_>) { - builder.info("Distcheck"); - let dir = builder.tempdir().join("distcheck"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - // Guarantee that these are built before we begin running. - builder.ensure(dist::PlainSourceTarball); - builder.ensure(dist::Src); - - command("tar") - .arg("-xf") - .arg(builder.ensure(dist::PlainSourceTarball).tarball()) - .arg("--strip-components=1") - .current_dir(&dir) - .run(builder); - command("./configure") - .args(&builder.config.configure_args) - .arg("--enable-vendor") - .current_dir(&dir) - .run(builder); - command(helpers::make(&builder.config.build.triple)) - .arg("check") - .current_dir(&dir) - .run(builder); - - // Now make sure that rust-src has all of libstd's dependencies - builder.info("Distcheck rust-src"); - let dir = builder.tempdir().join("distcheck-src"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - command("tar") - .arg("-xf") - .arg(builder.ensure(dist::Src).tarball()) - .arg("--strip-components=1") - .current_dir(&dir) - .run(builder); - - let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); - command(&builder.initial_cargo) - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("generate-lockfile") - .arg("--manifest-path") - .arg(&toml) - .current_dir(&dir) - .run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Bootstrap; - -impl Step for Bootstrap { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Tests the build system itself. - fn run(self, builder: &Builder<'_>) { - let host = builder.config.build; - let compiler = builder.compiler(0, host); - let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); - - // Some tests require cargo submodule to be present. - builder.build.require_submodule("src/tools/cargo", None); - - let mut check_bootstrap = command(builder.python()); - check_bootstrap - .args(["-m", "unittest", "bootstrap_test.py"]) - .env("BUILD_DIR", &builder.out) - .env("BUILD_PLATFORM", builder.build.build.triple) - .env("BOOTSTRAP_TEST_RUSTC_BIN", &builder.initial_rustc) - .env("BOOTSTRAP_TEST_CARGO_BIN", &builder.initial_cargo) - .current_dir(builder.src.join("src/bootstrap/")); - // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. - // Use `python -m unittest` manually if you want to pass arguments. - check_bootstrap.delay_failure().run(builder); - - let mut cmd = command(&builder.initial_cargo); - cmd.arg("test") - .args(["--features", "bootstrap-self-test"]) - .current_dir(builder.src.join("src/bootstrap")) - .env("RUSTFLAGS", "-Cdebuginfo=2") - .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) - .env("RUSTC_BOOTSTRAP", "1") - .env("RUSTDOC", builder.rustdoc(compiler)) - .env("RUSTC", &builder.initial_rustc); - if let Some(flags) = option_env!("RUSTFLAGS") { - // Use the same rustc flags for testing as for "normal" compilation, - // so that Cargo doesn’t recompile the entire dependency graph every time: - // https://github.com/rust-lang/rust/issues/49215 - cmd.env("RUSTFLAGS", flags); - } - // bootstrap tests are racy on directory creation so just run them one at a time. - // Since there's not many this shouldn't be a problem. - run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TierCheck { - pub compiler: Compiler, -} - -impl Step for TierCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/tier-check") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = - run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); - run.builder.ensure(TierCheck { compiler }); - } - - /// Tests the Platform Support page in the rustc book. - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.compiler.host, - Kind::Run, - "src/tools/tier-check", - SourceType::InTree, - &[], - ); - cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); - cargo.arg(builder.rustc(self.compiler)); - if builder.is_verbose() { - cargo.arg("--verbose"); - } - - let _guard = builder.msg( - Kind::Test, - self.compiler.stage, - "platform support check", - self.compiler.host, - self.compiler.host, - ); - BootstrapCommand::from(cargo).delay_failure().run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct LintDocs { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LintDocs { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/lint-docs") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LintDocs { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Tests that the lint examples in the rustc book generate the correct - /// lints and have the expected format. - fn run(self, builder: &Builder<'_>) { - builder.ensure(crate::core::build_steps::doc::RustcBook { - compiler: self.compiler, - target: self.target, - validate: true, - }); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustInstaller; - -impl Step for RustInstaller { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Ensure the version placeholder replacement tool builds - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - Kind::Test, - "src/tools/rust-installer", - SourceType::InTree, - &[], - ); - - let _guard = builder.msg( - Kind::Test, - compiler.stage, - "rust-installer", - bootstrap_host, - bootstrap_host, - ); - run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); - - // We currently don't support running the test.sh script outside linux(?) environments. - // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a - // set of scripts, which will likely allow dropping this if. - if bootstrap_host != "x86_64-unknown-linux-gnu" { - return; - } - - let mut cmd = command(builder.src.join("src/tools/rust-installer/test.sh")); - let tmpdir = testdir(builder, compiler.host).join("rust-installer"); - let _ = std::fs::remove_dir_all(&tmpdir); - let _ = std::fs::create_dir_all(&tmpdir); - cmd.current_dir(&tmpdir); - cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); - cmd.env("CARGO", &builder.initial_cargo); - cmd.env("RUSTC", &builder.initial_rustc); - cmd.env("TMP_DIR", &tmpdir); - cmd.delay_failure().run(builder); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-installer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TestHelpers { - pub target: TargetSelection, -} - -impl Step for TestHelpers { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("tests/auxiliary/rust_test_helpers.c") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TestHelpers { target: run.target }) - } - - /// Compiles the `rust_test_helpers.c` library which we used in various - /// `run-pass` tests for ABI testing. - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - // The x86_64-fortanix-unknown-sgx target doesn't have a working C - // toolchain. However, some x86_64 ELF objects can be linked - // without issues. Use this hack to compile the test helpers. - let target = if self.target == "x86_64-fortanix-unknown-sgx" { - TargetSelection::from_user("x86_64-unknown-linux-gnu") - } else { - self.target - }; - let dst = builder.test_helpers_out(target); - let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); - if up_to_date(&src, &dst.join("librust_test_helpers.a")) { - return; - } - - let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); - t!(fs::create_dir_all(&dst)); - let mut cfg = cc::Build::new(); - - // We may have found various cross-compilers a little differently due to our - // extra configuration, so inform cc of these compilers. Note, though, that - // on MSVC we still need cc's detection of env vars (ugh). - if !target.is_msvc() { - if let Some(ar) = builder.ar(target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(target)); - } - cfg.cargo_metadata(false) - .out_dir(&dst) - .target(&target.triple) - .host(&builder.config.build.triple) - .opt_level(0) - .warnings(false) - .debug(false) - .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) - .compile("rust_test_helpers"); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenCranelift { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for CodegenCranelift { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); - - if builder.doc_tests == DocTests::Only { - return; - } - - if builder.download_rustc() { - builder.info("CI rustc uses the default codegen backend. skipping"); - return; - } - - if !target_supports_cranelift_backend(run.target) { - builder.info("target not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if builder.remote_tested(run.target) { - builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if !builder.config.codegen_backends(run.target).contains(&"cranelift".to_owned()) { - builder.info("cranelift not in rust.codegen-backends. skipping"); - return; - } - - builder.ensure(CodegenCranelift { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - builder.ensure(compile::Std::new(compiler, target)); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let build_cargo = || { - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works - SourceType::InTree, - target, - Kind::Run, - ); - - cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); - cargo - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); - compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - // Avoid incremental cache issues when changing rustc - cargo.env("CARGO_BUILD_INCREMENTAL", "false"); - - cargo - }; - - builder.info(&format!( - "{} cranelift stage{} ({} -> {})", - Kind::Test.description(), - compiler.stage, - &compiler.host, - target - )); - let _time = helpers::timeit(builder); - - // FIXME handle vendoring for source tarballs before removing the --skip-test below - let download_dir = builder.out.join("cg_clif_download"); - - // FIXME: Uncomment the `prepare` command below once vendoring is implemented. - /* - let mut prepare_cargo = build_cargo(); - prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); - #[allow(deprecated)] - builder.config.try_run(&mut prepare_cargo.into()).unwrap(); - */ - - let mut cargo = build_cargo(); - cargo - .arg("--") - .arg("test") - .arg("--download-dir") - .arg(&download_dir) - .arg("--out-dir") - .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) - .arg("--no-unstable-features") - .arg("--use-backend") - .arg("cranelift") - // Avoid having to vendor the standard library dependencies - .arg("--sysroot") - .arg("llvm") - // These tests depend on crates that are not yet vendored - // FIXME remove once vendoring is handled - .arg("--skip-test") - .arg("testsuite.extended_sysroot"); - cargo.args(builder.config.test_args()); - - cargo.into_cmd().run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenGCC { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for CodegenGCC { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); - - if builder.doc_tests == DocTests::Only { - return; - } - - if builder.download_rustc() { - builder.info("CI rustc uses the default codegen backend. skipping"); - return; - } - - let triple = run.target.triple; - let target_supported = - if triple.contains("linux") { triple.contains("x86_64") } else { false }; - if !target_supported { - builder.info("target not supported by rustc_codegen_gcc. skipping"); - return; - } - - if builder.remote_tested(run.target) { - builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); - return; - } - - if !builder.config.codegen_backends(run.target).contains(&"gcc".to_owned()) { - builder.info("gcc not in rust.codegen-backends. skipping"); - return; - } - - builder.ensure(CodegenGCC { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - builder.ensure(compile::Std::new_with_extra_rust_args(compiler, target, &[ - "-Csymbol-mangling-version=v0", - "-Cpanic=abort", - ])); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let build_cargo = || { - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works - SourceType::InTree, - target, - Kind::Run, - ); - - cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); - cargo - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); - compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - // Avoid incremental cache issues when changing rustc - cargo.env("CARGO_BUILD_INCREMENTAL", "false"); - cargo.rustflag("-Cpanic=abort"); - - cargo - }; - - builder.info(&format!( - "{} GCC stage{} ({} -> {})", - Kind::Test.description(), - compiler.stage, - &compiler.host, - target - )); - let _time = helpers::timeit(builder); - - // FIXME: Uncomment the `prepare` command below once vendoring is implemented. - /* - let mut prepare_cargo = build_cargo(); - prepare_cargo.arg("--").arg("prepare"); - #[allow(deprecated)] - builder.config.try_run(&mut prepare_cargo.into()).unwrap(); - */ - - let mut cargo = build_cargo(); - - cargo - .arg("--") - .arg("test") - .arg("--use-system-gcc") - .arg("--use-backend") - .arg("gcc") - .arg("--out-dir") - .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) - .arg("--release") - .arg("--mini-tests") - .arg("--std-tests"); - cargo.args(builder.config.test_args()); - - cargo.into_cmd().run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TestFloatParse { - path: PathBuf, - host: TargetSelection, -} - -impl Step for TestFloatParse { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/etc/test-float-parse") - } - - fn make_run(run: RunConfig<'_>) { - for path in run.paths { - let path = path.assert_single_path().path.clone(); - run.builder.ensure(Self { path, host: run.target }); - } - } - - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(builder.top_stage, bootstrap_host); - let path = self.path.to_str().unwrap(); - let crate_name = self.path.components().last().unwrap().as_os_str().to_str().unwrap(); - - builder.ensure(tool::TestFloatParse { host: self.host }); - - // Run any unit tests in the crate - let cargo_test = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - bootstrap_host, - Kind::Test, - path, - SourceType::InTree, - &[], - ); - - run_cargo_test( - cargo_test, - &[], - &[], - crate_name, - crate_name, - compiler, - bootstrap_host, - builder, - ); - - // Run the actual parse tests. - let mut cargo_run = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - bootstrap_host, - Kind::Run, - path, - SourceType::InTree, - &[], - ); - - cargo_run.arg("--"); - if builder.config.args().is_empty() { - // By default, exclude tests that take longer than ~1m. - cargo_run.arg("--skip-huge"); - } else { - cargo_run.args(builder.config.args()); - } +/// This combined step runs the specified tests (or all of `tests/coverage`) +/// in both "coverage-map" and "coverage-run" modes. +/// +/// Used by: +/// - `x test coverage` +/// - `x test tests/coverage` +/// - `x test tests/coverage/trivial.rs` (etc) +/// +/// (Each individual mode also has its own step that will run the tests in +/// just that mode.) +// Runs `tests/coverage` in "coverage-map" mode only. +// Used by `x test` and `x test coverage-map`. +coverage_test_alias!(CoverageMap { + alias_and_mode: "coverage-map", + default: true, + only_hosts: false, +}); +// Runs `tests/coverage` in "coverage-run" mode only. +// Used by `x test` and `x test coverage-run`. +coverage_test_alias!(CoverageRun { + alias_and_mode: "coverage-run", + default: true, + // Compiletest knows how to automatically skip these tests when cross-compiling, + // but skipping the whole step here makes it clearer that they haven't run at all. + only_hosts: true, +}); - cargo_run.into_cmd().run(builder); - } -} +host_test!(CoverageRunRustdoc { + path: "tests/coverage-run-rustdoc", + mode: "coverage-run", + suite: "coverage-run-rustdoc" +}); diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/book_test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/book_test.rs new file mode 100644 index 00000000..e05119f3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/book_test.rs @@ -0,0 +1,110 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct BookTest { + compiler: Compiler, + path: PathBuf, + name: &'static str, + is_ext_doc: bool, +} + +impl Step for BookTest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Runs the documentation tests for a book in `src/doc`. + /// + /// This uses the `rustdoc` that sits next to `compiler`. + fn run(self, builder: &Builder<'_>) { + // External docs are different from local because: + // - Some books need pre-processing by mdbook before being tested. + // - They need to save their state to toolstate. + // - They are only tested on the "checktools" builders. + // + // The local docs are testedByDefault, and we don't want to pay the + // cost of building mdbook, so they use `rustdoc --test` directly. + // Also, the unstable book is special because SUMMARY.md is generated, + // so it is easier to just run `rustdoc` on its files. + if self.is_ext_doc { + self.run_ext_doc(builder); + } else { + self.run_local_doc(builder); + } + } +} + +impl BookTest { + /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) + /// which in turn runs `rustdoc --test` on each file in the book. + fn run_ext_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // mdbook just executes a binary named "rustdoc", so we need to update + // PATH so that it points to our rustdoc. + let mut rustdoc_path = builder.rustdoc(compiler); + rustdoc_path.pop(); + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) + .expect("could not add rustdoc to PATH"); + + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let path = builder.src.join(&self.path); + // Books often have feature-gated example text. + rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); + rustbook_cmd.env("PATH", new_path).arg("test").arg(path); + builder.add_rust_test_threads(&mut rustbook_cmd); + let _guard = builder.msg( + Kind::Test, + compiler.stage, + format_args!("mdbook {}", self.path.display()), + compiler.host, + compiler.host, + ); + let _time = helpers::timeit(builder); + let toolstate = if rustbook_cmd.delay_failure().run(builder) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate(self.name, toolstate); + } + + /// This runs `rustdoc --test` on all `.md` files in the path. + fn run_local_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let host = self.compiler.host; + + builder.ensure(compile::Std::new(compiler, host)); + + let _guard = + builder.msg(Kind::Test, compiler.stage, format!("book {}", self.name), host, host); + + // Do a breadth-first traversal of the `src/doc` directory and just run + // tests for all files that end in `*.md` + let mut stack = vec![builder.src.join(self.path)]; + let _time = helpers::timeit(builder); + let mut files = Vec::new(); + while let Some(p) = stack.pop() { + if p.is_dir() { + stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); + continue; + } + + if p.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + + files.push(p); + } + + files.sort(); + + for file in files { + markdown_test(builder, compiler, &file); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/bootstrap.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/bootstrap.rs new file mode 100644 index 00000000..afded4f6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/bootstrap.rs @@ -0,0 +1,57 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Bootstrap; + +impl Step for Bootstrap { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Tests the build system itself. + fn run(self, builder: &Builder<'_>) { + let host = builder.config.build; + let compiler = builder.compiler(0, host); + let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); + + // Some tests require cargo submodule to be present. + builder.build.require_submodule("src/tools/cargo", None); + + let mut check_bootstrap = command(builder.python()); + check_bootstrap + .args(["-m", "unittest", "bootstrap_test.py"]) + .env("BUILD_DIR", &builder.out) + .env("BUILD_PLATFORM", builder.build.build.triple) + .env("BOOTSTRAP_TEST_RUSTC_BIN", &builder.initial_rustc) + .env("BOOTSTRAP_TEST_CARGO_BIN", &builder.initial_cargo) + .current_dir(builder.src.join("src/bootstrap/")); + // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. + // Use `python -m unittest` manually if you want to pass arguments. + check_bootstrap.delay_failure().run(builder); + + let mut cmd = command(&builder.initial_cargo); + cmd.arg("test") + .args(["--features", "bootstrap-self-test"]) + .current_dir(builder.src.join("src/bootstrap")) + .env("RUSTFLAGS", "-Cdebuginfo=2") + .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("RUSTC_BOOTSTRAP", "1") + .env("RUSTDOC", builder.rustdoc(compiler)) + .env("RUSTC", &builder.initial_rustc); + if let Some(flags) = option_env!("RUSTFLAGS") { + // Use the same rustc flags for testing as for "normal" compilation, + // so that Cargo doesn’t recompile the entire dependency graph every time: + // https://github.com/rust-lang/rust/issues/49215 + cmd.env("RUSTFLAGS", flags); + } + // bootstrap tests are racy on directory creation so just run them one at a time. + // Since there's not many this shouldn't be a problem. + run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo.rs new file mode 100644 index 00000000..7f630bbd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo.rs @@ -0,0 +1,68 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Cargo { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for Cargo { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargo") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Cargo { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for `cargo` packaged with Rust. + fn run(self, builder: &Builder<'_>) { + let compiler = self.common.compiler; + let host = self.common.host; + + builder.ensure(tool::Cargo { compiler, target: self.common.host }); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + self.common.host, + Kind::Test, + "src/tools/cargo", + SourceType::Submodule, + &[], + ); + + // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` + let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.common.host, builder); + + // Don't run cross-compile tests, we may not have cross-compiled libstd libs + // available. + cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); + // Forcibly disable tests using nightly features since any changes to + // those features won't be able to land. + cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); + cargo.env("PATH", path_for_cargo(builder, compiler)); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: vec!["cargo".into()], + target: self.common.host.triple.to_string(), + host: self.common.host.triple.to_string(), + stage: self.common.stage, + }, + builder, + ); + + let _time = helpers::timeit(builder); + add_flags_and_try_run_tests(builder, &mut cargo); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo_miri.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo_miri.rs new file mode 100644 index 00000000..2795a173 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargo_miri.rs @@ -0,0 +1,75 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CargoMiri { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CargoMiri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri/cargo-miri") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CargoMiri { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); + } + + /// Tests `cargo miri test`. + fn run(self, builder: &Builder<'_>) { + let host = self.common.host; + let target = self.common.target; + let stage = self.common.stage; + if stage == 0 { + eprintln!("cargo-miri cannot be tested at stage 0"); + std::process::exit(1); + } + + // This compiler runs on the host, we'll just use it for the target. + let compiler = builder.compiler(stage, host); + + // Run `cargo miri test`. + // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures + // that we get the desired output), but that is sufficient to make sure that the libtest harness + // itself executes properly under Miri, and that all the logic in `cargo-miri` does not explode. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, // it's unclear what to use here, we're not building anything just doing a smoke test! + target, + Kind::MiriTest, + "src/tools/miri/test-cargo-miri", + SourceType::Submodule, + &[], + ); + + // We're not using `prepare_cargo_test` so we have to do this ourselves. + // (We're not using that as the test-cargo-miri crate is not known to bootstrap.) + match builder.doc_tests { + DocTests::Yes => {} + DocTests::No => { + cargo.args(["--lib", "--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Only => { + cargo.arg("--doc"); + } + } + + // Finally, pass test-args and run everything. + cargo.arg("--").args(builder.config.test_args()); + let mut cargo = BootstrapCommand::from(cargo); + { + let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "cargo-miri", host, target); + let _time = helpers::timeit(builder); + cargo.run(builder); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/cargotest.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargotest.rs new file mode 100644 index 00000000..a8f3581d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/cargotest.rs @@ -0,0 +1,52 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Cargotest { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for Cargotest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargotest") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Cargotest { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will check out a few Rust projects and run `cargo + /// test` to ensure that we don't regress the test suites there. + fn run(self, builder: &Builder<'_>) { + let compiler = self.common.compiler; + let host = self.common.host; + builder.ensure(compile::Rustc::new(compiler, compiler.host)); + let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); + + // Note that this is a short, cryptic, and not scoped directory name. This + // is currently to minimize the length of path on Windows where we otherwise + // quickly run into path name limit constraints. + let out_dir = builder.out.join("ct"); + t!(fs::create_dir_all(&out_dir)); + + let _time = helpers::timeit(builder); + let mut cmd = builder.tool_cmd(Tool::CargoTest); + cmd.arg(&cargo) + .arg(&out_dir) + .args(builder.config.test_args()) + .env("RUSTC", builder.rustc(compiler)) + .env("RUSTDOC", builder.rustdoc(compiler)); + add_rustdoc_cargo_linker_args(&mut cmd, builder, compiler.host, LldThreads::No); + cmd.delay_failure().run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/check_if_tidy_is_installed.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/check_if_tidy_is_installed.rs new file mode 100644 index 00000000..bacd8c1c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/check_if_tidy_is_installed.rs @@ -0,0 +1,3 @@ +fn check_if_tidy_is_installed(builder: &Builder<'_>) -> bool { + command("tidy").allow_failure().arg("--version").run_capture_stdout(builder).is_success() +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/clippy.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/clippy.rs new file mode 100644 index 00000000..c522c567 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/clippy.rs @@ -0,0 +1,65 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Clippy { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for Clippy { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/clippy") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Clippy { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for clippy. + fn run(self, builder: &Builder<'_>) { + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; + + builder.ensure(tool::Clippy { compiler, target: self.common.host, extra_features: Vec::new() }); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + Kind::Test, + "src/tools/clippy", + SourceType::InTree, + &[], + ); + + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); + cargo.env("HOST_LIBS", host_libs); + + cargo.add_rustc_lib_path(builder); + let cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); + + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); + + // Clippy reports errors if it blessed the outputs + if cargo.allow_failure().run(builder) { + // The tests succeeded; nothing to do. + return; + } + + if !builder.config.cmd.bless() { + crate::exit!(1); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_cranelift.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_cranelift.rs new file mode 100644 index 00000000..fc045c7b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_cranelift.rs @@ -0,0 +1,130 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenCranelift { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CodegenCranelift { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + if builder.download_rustc() { + builder.info("CI rustc uses the default codegen backend. skipping"); + return; + } + + if !target_supports_cranelift_backend(run.target) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if !builder.config.codegen_backends(run.target).contains(&"cranelift".to_owned()) { + builder.info("cranelift not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenCranelift { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.common.compiler; + let target = self.common.target; + + builder.ensure(compile::Std::new(compiler, target)); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + Kind::Run, + ); + + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + + cargo + }; + + builder.info(&format!( + "{} cranelift stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(builder); + + // FIXME handle vendoring for source tarballs before removing the --skip-test below + let download_dir = builder.out.join("cg_clif_download"); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + cargo + .arg("--") + .arg("test") + .arg("--download-dir") + .arg(&download_dir) + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) + .arg("--no-unstable-features") + .arg("--use-backend") + .arg("cranelift") + // Avoid having to vendor the standard library dependencies + .arg("--sysroot") + .arg("llvm") + // These tests depend on crates that are not yet vendored + // FIXME remove once vendoring is handled + .arg("--skip-test") + .arg("testsuite.extended_sysroot"); + cargo.args(builder.config.test_args()); + + cargo.into_cmd().run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_gcc.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_gcc.rs new file mode 100644 index 00000000..0db352d6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/codegen_gcc.rs @@ -0,0 +1,129 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenGCC { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CodegenGCC { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + if builder.download_rustc() { + builder.info("CI rustc uses the default codegen backend. skipping"); + return; + } + + let triple = run.target.triple; + let target_supported = + if triple.contains("linux") { triple.contains("x86_64") } else { false }; + if !target_supported { + builder.info("target not supported by rustc_codegen_gcc. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); + return; + } + + if !builder.config.codegen_backends(run.target).contains(&"gcc".to_owned()) { + builder.info("gcc not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenGCC { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.common.compiler; + let target = self.common.target; + + builder.ensure(compile::Std::new_with_extra_rust_args(compiler, target, &[ + "-Csymbol-mangling-version=v0", + "-Cpanic=abort", + ])); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + Kind::Run, + ); + + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + cargo.rustflag("-Cpanic=abort"); + + cargo + }; + + builder.info(&format!( + "{} GCC stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(builder); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare"); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + + cargo + .arg("--") + .arg("test") + .arg("--use-system-gcc") + .arg("--use-backend") + .arg("gcc") + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) + .arg("--release") + .arg("--mini-tests") + .arg("--std-tests"); + cargo.args(builder.config.test_args()); + + cargo.into_cmd().run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest.rs new file mode 100644 index 00000000..8a7bfd27 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest.rs @@ -0,0 +1,229 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Compiletest { + compiler: Compiler, + target: TargetSelection, + mode: &'static str, + suite: &'static str, + path: &'static str, + compare_mode: Option<&'static str>, +} + +impl Step for Compiletest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Executes the `compiletest` tool to run a suite of tests. + /// + /// Compiles all tests with `compiler` for `target` with the specified + /// compiletest `mode` and `suite` arguments. For example `mode` can be + /// "run-pass" or `suite` can be something like `debuginfo`. + fn run(self, builder: &Builder<'_>) { + if builder.doc_tests == DocTests::Only { + return; + } + + if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { + eprintln!( + "ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail\nHELP: to test the compiler, use `--stage 1` instead\nHELP: to test the standard library, use `--stage 0 library/std` instead\nNOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." + ); + crate::exit!(1); + } + + let mut compiler = self.compiler; + let target = self.target; + let mode = self.mode; + let suite = self.suite; + + // Path for test suite + let suite_path = self.path; + + // Skip codegen tests if they aren't enabled in configuration. + if !builder.config.codegen_tests && suite == "codegen" { + return; + } + + // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most + // part test the *API* of the compiler, not how it compiles a given file. As a result, we + // can run them against the stage 1 sources as long as we build them with the stage 0 + // bootstrap compiler. + // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the + // running compiler in stage 2 when plugins run. + let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { + // At stage 0 (stage - 1) we are using the beta compiler. Using `self.target` can lead finding + // an incorrect compiler path on cross-targets, as the stage 0 beta compiler is always equal + // to `build.build` in the configuration. + let build = builder.build.build; + + compiler = builder.compiler(compiler.stage - 1, build); + format!("stage{}-{}", compiler.stage + 1, build) + } else { + format!("stage{}-{}", compiler.stage, target) + }; + + if suite.ends_with("fulldeps") { + builder.ensure(compile::Rustc::new(compiler, target)); + } + + if suite == "debuginfo" { + builder.ensure(dist::DebuggerScripts { + sysroot: builder.sysroot(compiler).to_path_buf(), + host: target, + }); + } + + // Also provide `rust_test_helpers` for the host. + builder.ensure(TestHelpers { target: compiler.host }); + + // ensure that `libproc_macro` is available on the host. + if suite == "mir-opt" { + builder.ensure(compile::Std::new_for_mir_opt_tests(compiler, compiler.host)); + } else { + builder.ensure(compile::Std::new(compiler, compiler.host)); + } + + // As well as the target + if suite != "mir-opt" { + builder.ensure(TestHelpers { target }); + } + + let mut cmd = builder.tool_cmd(Tool::Compiletest); + + if suite == "mir-opt" { + builder.ensure(compile::Std::new_for_mir_opt_tests(compiler, target)); + } else { + builder.ensure(compile::Std::new(compiler, target)); + } + + builder.ensure(RemoteCopyLibs { compiler, target }); + + // compiletest currently has... a lot of arguments, so let's just pass all + // of them! + + cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); + cmd.arg("--run-lib-path").arg(builder.sysroot_target_libdir(compiler, target)); + cmd.arg("--rustc-path").arg(builder.rustc(compiler)); + + // Minicore auxiliary lib for `no_core` tests that need `core` stubs in cross-compilation + // scenarios. + cmd.arg("--minicore-path") + .arg(builder.src.join("tests").join("auxiliary").join("minicore.rs")); + + let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); + + if mode == "run-make" { + let cargo_path = if builder.top_stage == 0 { + // If we're using `--stage 0`, we should provide the bootstrap cargo. + builder.initial_cargo.clone() + } else { + // We need to properly build cargo using the suitable stage compiler. + + let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(|| + // HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if + // you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built + // and produce a cargo built with stage 2 rustc. To fix this, we need to chop off + // the compiler stage by 1 to align with expected `./x test run-make --stage N` + // behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri + // which does a similar hack. + builder.compiler(builder.top_stage - 1, compiler.host)); + + builder.ensure(tool::Cargo { compiler, target: compiler.host }) + }; + + cmd.arg("--cargo-path").arg(cargo_path); + } + + // Avoid depending on rustdoc when we don't need it. + if mode == "rustdoc" + || mode == "run-make" + || (mode == "ui" && is_rustdoc) + || mode == "js-doc-test" + || mode == "rustdoc-json" + || suite == "coverage-run-rustdoc" + { + cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); + } + + if mode == "rustdoc-json" { + // Use the beta compiler for jsondocck + let json_compiler = compiler.with_stage(0); + cmd.arg("--jsondocck-path") + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); + cmd.arg("--jsondoclint-path") + .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); + } + + if matches!(mode, "coverage-map" | "coverage-run") { + let coverage_dump = builder.tool_exe(Tool::CoverageDump); + cmd.arg("--coverage-dump-path").arg(coverage_dump); + } + + cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); + cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); + + // When top stage is 0, that means that we're testing an externally provided compiler. + // In that case we need to use its specific sysroot for tests to pass. + let sysroot = if builder.top_stage == 0 { + builder.initial_sysroot.clone() + } else { + builder.sysroot(compiler).to_path_buf() + }; + cmd.arg("--sysroot-base").arg(sysroot); + cmd.arg("--stage-id").arg(stage_id); + cmd.arg("--suite").arg(suite); + cmd.arg("--mode").arg(mode); + cmd.arg("--target").arg(target.rustc_target_arg()); + cmd.arg("--host").arg(&*compiler.host.triple); + cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); + + if builder.build.config.llvm_enzyme { + cmd.arg("--has-enzyme"); + } + + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + + if builder.config.cmd.force_rerun() { + cmd.arg("--force-rerun"); + } + + let compare_mode = + builder.config.cmd.compare_mode().or_else(|| { + if builder.config.test_compare_mode { self.compare_mode } else { None } + }); + + if let Some(ref pass) = builder.config.cmd.pass() { + cmd.arg("--pass"); + cmd.arg(pass); + } + + if let Some(ref run) = builder.config.cmd.run() { + cmd.arg("--run"); + cmd.arg(run); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } else if mode == "js-doc-test" { + panic!("need nodejs to run js-doc-test suite"); + } + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + if builder.config.rust_optimize_tests { + cmd.arg("--optimize-tests"); + } + if builder.config.rust_randomize_layout { + cmd.arg("--rust-randomized-layout"); + } + if builder.config.cmd.only_modified() { + cmd.arg("--only-modified"); + } + if let Some(compiletest_diff_tool) = &builder.config.compiletest_diff_tool { + cmd.arg("--compiletest-diff-tool").arg(compiletest_diff_tool); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest_test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest_test.rs new file mode 100644 index 00000000..5a9b9df2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/compiletest_test.rs @@ -0,0 +1,57 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CompiletestTest { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CompiletestTest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/compiletest") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CompiletestTest { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for compiletest. + fn run(self, builder: &Builder<'_>) { + let host = self.common.host; + let compiler = self.common.compiler; + + // We need `ToolStd` for the locally-built sysroot because + // compiletest uses unstable features of the `test` crate. + builder.ensure(compile::Std::new(compiler, host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + // compiletest uses libtest internals; make it use the in-tree std to make sure it never breaks + // when std sources change. + Mode::ToolStd, + host, + Kind::Test, + "src/tools/compiletest", + SourceType::InTree, + &[], + ); + cargo.allow_features("test"); + run_cargo_test( + cargo, + &[], + &[], + "compiletest", + "compiletest self test", + compiler, + host, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/coverage.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/coverage.rs new file mode 100644 index 00000000..8d430220 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/coverage.rs @@ -0,0 +1,66 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Coverage { + pub common: common_test_fields::CommonTestFields, +} + +impl Coverage { + const PATH: &'static str = "tests/coverage"; + const SUITE: &'static str = "coverage"; + + /// Runs the coverage test suite (or a user-specified subset) in one mode. + /// + /// This same function is used by the multi-mode step ([`Coverage`]) and by + /// the single-mode steps ([`CoverageMap`] and [`CoverageRun`]), to help + /// ensure that they all behave consistently with each other, regardless of + /// how the coverage tests have been invoked. + fn run_coverage_tests( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + mode: &'static str, + ) { + // Like many other test steps, we delegate to a `Compiletest` step to + // actually run the tests. (See `test_definitions!`.) + builder.ensure(Compiletest { + compiler, + target, + mode, + suite: Self::SUITE, + path: Self::PATH, + compare_mode: None, + }); + } +} + +impl Step for Coverage { + type Output = (); + /// We rely on the individual CoverageMap/CoverageRun steps to run themselves. + const DEFAULT: bool = false; + /// When manually invoked, try to run as much as possible. + /// Compiletest will automatically skip the "coverage-run" tests if necessary. + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Take responsibility for command-line paths within `tests/coverage`. + run.suite_path(Self::PATH) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure(Coverage { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + // Run the specified coverage tests (possibly all of them) in both modes. + Self::run_coverage_tests(builder, self.common.compiler, self.common.target, CoverageMap::MODE); + Self::run_coverage_tests(builder, self.common.compiler, self.common.target, CoverageRun::MODE); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate.rs new file mode 100644 index 00000000..25286b28 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate.rs @@ -0,0 +1,135 @@ +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Crate { + pub compiler: Compiler, + pub target: TargetSelection, + pub mode: Mode, + pub crates: Vec, +} + +impl Step for Crate { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); + } + + /// Runs all unit tests plus documentation tests for a given crate defined + /// by a `Cargo.toml` (single manifest) + /// + /// This is what runs tests for crates like the standard library, compiler, etc. + /// It essentially is the driver for running `cargo test`. + /// + /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` + /// arguments, and those arguments are discovered from `cargo metadata`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let mode = self.mode; + + // Prepare sysroot + // See [field@compile::Std::force_recompile]. + builder.ensure(compile::Std::force_recompile(compiler, compiler.host)); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let mut cargo = if builder.kind == Kind::Miri { + if builder.top_stage == 0 { + eprintln!("ERROR: `x.py miri` requires stage 1 or higher"); + std::process::exit(1); + } + + // Build `cargo miri test` command + // (Implicitly prepares target sysroot) + let mut cargo = builder::Cargo::new( + builder, + compiler, + mode, + SourceType::InTree, + target, + Kind::MiriTest, + ); + // This hack helps bootstrap run standard library tests in Miri. The issue is as + // follows: when running `cargo miri test` on libcore, cargo builds a local copy of core + // and makes it a dependency of the integration test crate. This copy duplicates all the + // lang items, so the build fails. (Regular testing avoids this because the sysroot is a + // literal copy of what `cargo build` produces, but since Miri builds its own sysroot + // this does not work for us.) So we need to make it so that the locally built libcore + // contains all the items from `core`, but does not re-define them -- we want to replace + // the entire crate but a re-export of the sysroot crate. We do this by swapping out the + // source file: if `MIRI_REPLACE_LIBRS_IF_NOT_TEST` is set and we are building a + // `lib.rs` file, and a `lib.miri.rs` file exists in the same folder, we build that + // instead. But crucially we only do that for the library, not the test builds. + cargo.env("MIRI_REPLACE_LIBRS_IF_NOT_TEST", "1"); + cargo + } else { + // Also prepare a sysroot for the target. + if builder.config.build != target { + builder.ensure(compile::Std::force_recompile(compiler, target)); + builder.ensure(RemoteCopyLibs { compiler, target }); + } + + // Build `cargo test` command + builder::Cargo::new(builder, compiler, mode, SourceType::InTree, target, builder.kind) + }; + + match mode { + Mode::Std => { + if builder.kind == Kind::Miri { + // We can't use `std_cargo` as that uses `optimized-compiler-builtins` which + // needs host tools for the given target. This is similar to what `compile::Std` + // does when `is_for_mir_opt_tests` is true. There's probably a chance for + // de-duplication here... `std_cargo` should support a mode that avoids needing + // host tools. + cargo + .arg("--manifest-path") + .arg(builder.src.join("library/sysroot/Cargo.toml")); + } else { + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + // `std-cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, + // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. + // Override it. + if builder.download_rustc() && compiler.stage > 0 { + let sysroot = builder + .out + .join(compiler.host) + .join(format!("stage{}-test-sysroot", compiler.stage)); + cargo.env("RUSTC_SYSROOT", sysroot); + } + } + } + Mode::Rustc => { + compile::rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); + } + _ => panic!("can only test libraries"), + }; + + run_cargo_test( + cargo, + &[], + &self.crates, + &self.crates[0], + &*crate_description(&self.crates), + compiler, + target, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_bootstrap.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_bootstrap.rs new file mode 100644 index 00000000..d0508db9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_bootstrap.rs @@ -0,0 +1,47 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateBootstrap { + path: PathBuf, + host: TargetSelection, +} + +impl Step for CrateBootstrap { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/jsondoclint") + .path("src/tools/suggest-tests") + .path("src/tools/replace-version-placeholder") + .alias("tidyselftest") + } + + fn make_run(run: RunConfig<'_>) { + for path in run.paths { + let path = path.assert_single_path().path.clone(); + run.builder.ensure(CrateBootstrap { host: run.target, path }); + } + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let mut path = self.path.to_str().unwrap(); + if path == "tidyselftest" { + path = "src/tools/tidy"; + } + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + Kind::Test, + path, + SourceType::InTree, + &[], + ); + let crate_name = path.rsplit_once('/').unwrap().1; + run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_build_helper.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_build_helper.rs new file mode 100644 index 00000000..3bbc3cf8 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_build_helper.rs @@ -0,0 +1,53 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateBuildHelper { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CrateBuildHelper { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/build_helper") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CrateBuildHelper { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for build_helper. + fn run(self, builder: &Builder<'_>) { + let host = self.common.host; + let compiler = self.common.compiler; + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + host, + Kind::Test, + "src/build_helper", + SourceType::InTree, + &[], + ); + cargo.allow_features("test"); + run_cargo_test( + cargo, + &[], + &[], + "build_helper", + "build_helper self test", + compiler, + host, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_librustc.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_librustc.rs new file mode 100644 index 00000000..69cedf2d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_librustc.rs @@ -0,0 +1,36 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateLibrustc { + compiler: Compiler, + target: TargetSelection, + crates: Vec, +} + +impl Step for CrateLibrustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run.make_run_crates(Alias::Compiler); + + builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.target)); + + builder.ensure(Crate { + compiler: self.compiler, + target: self.target, + mode: Mode::Rustc, + crates: self.crates, + }); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_run_make_support.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_run_make_support.rs new file mode 100644 index 00000000..04298145 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_run_make_support.rs @@ -0,0 +1,53 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateRunMakeSupport { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for CrateRunMakeSupport { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/run-make-support") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CrateRunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for run-make-support. + fn run(self, builder: &Builder<'_>) { + let host = self.common.host; + let compiler = self.common.compiler; + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + host, + Kind::Test, + "src/tools/run-make-support", + SourceType::InTree, + &[], + ); + cargo.allow_features("test"); + run_cargo_test( + cargo, + &[], + &[], + "run-make-support", + "run-make-support self test", + compiler, + host, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc.rs new file mode 100644 index 00000000..77e8cc91 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc.rs @@ -0,0 +1,70 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdoc { + host: TargetSelection, +} + +impl Step for CrateRustdoc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["src/librustdoc", "src/tools/rustdoc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdoc { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + let compiler = if builder.download_rustc() { + builder.compiler(builder.top_stage, target) + } else { + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + builder.compiler_for(builder.top_stage, target, target) + }; + builder.ensure(compile::Std::new(compiler, target)); + builder.ensure(compile::Rustc::new(compiler, target)); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind, + "src/tools/rustdoc", + SourceType::InTree, + &[], + ); + if self.host.contains("musl") { + cargo.arg("-Ctarget-feature=-crt-static"); + } + + let libdir = if builder.download_rustc() { + builder.rustc_libdir(compiler) + } else { + builder.sysroot_target_libdir(compiler, target).to_path_buf() + }; + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*libdir)); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + run_cargo_test( + cargo, + &[], + &["rustdoc:0.0.0".to_string()], + "rustdoc", + "rustdoc", + compiler, + target, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc_json_types.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc_json_types.rs new file mode 100644 index 00000000..7b2933e1 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/crate_rustdoc_json_types.rs @@ -0,0 +1,60 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdocJsonTypes { + host: TargetSelection, +} + +impl Step for CrateRustdocJsonTypes { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/rustdoc-json-types") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdocJsonTypes { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + let compiler = builder.compiler_for(builder.top_stage, target, target); + builder.ensure(compile::Rustc::new(compiler, target)); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind, + "src/rustdoc-json-types", + SourceType::InTree, + &[], + ); + + // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. + let libtest_args = if self.host.contains("musl") { + ["-Ctarget-feature=crt-static"].as_slice() + } else { + &[] + }; + + run_cargo_test( + cargo, + libtest_args, + &["rustdoc-json-types".to_string()], + "rustdoc-json-types", + "rustdoc-json-types", + compiler, + target, + builder, + ); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/distcheck.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/distcheck.rs new file mode 100644 index 00000000..3dbd036b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/distcheck.rs @@ -0,0 +1,66 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Distcheck; + +impl Step for Distcheck { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("distcheck") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Distcheck); + } + + /// Runs "distcheck", a 'make check' from a tarball + fn run(self, builder: &Builder<'_>) { + builder.info("Distcheck"); + let dir = builder.tempdir().join("distcheck"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + // Guarantee that these are built before we begin running. + builder.ensure(dist::PlainSourceTarball); + builder.ensure(dist::Src); + + command("tar") + .arg("-xf") + .arg(builder.ensure(dist::PlainSourceTarball).tarball()) + .arg("--strip-components=1") + .current_dir(&dir) + .run(builder); + command("./configure") + .args(&builder.config.configure_args) + .arg("--enable-vendor") + .current_dir(&dir) + .run(builder); + command(helpers::make(&builder.config.build.triple)) + .arg("check") + .current_dir(&dir) + .run(builder); + + // Now make sure that rust-src has all of libstd's dependencies + builder.info("Distcheck rust-src"); + let dir = builder.tempdir().join("distcheck-src"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + command("tar") + .arg("-xf") + .arg(builder.ensure(dist::Src).tarball()) + .arg("--strip-components=1") + .current_dir(&dir) + .run(builder); + + let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); + command(&builder.initial_cargo) + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .arg("generate-lockfile") + .arg("--manifest-path") + .arg(&toml) + .current_dir(&dir) + .run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/error_index.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/error_index.rs new file mode 100644 index 00000000..ae233861 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/error_index.rs @@ -0,0 +1,57 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ErrorIndex { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // error_index_generator depends on librustdoc. Use the compiler that + // is normally used to build rustdoc for other tests (like compiletest + // tests in tests/rustdoc) so that it shares the same artifacts. + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(ErrorIndex { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); + } + + /// Runs the error index generator tool to execute the tests located in the error + /// index. + /// + /// The `error_index_generator` tool lives in `src/tools` and is used to + /// generate a markdown file from the error indexes of the code base which is + /// then passed to `rustdoc --test`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.common.compiler; + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + let output = dir.join("error-index.md"); + + let mut tool = tool::ErrorIndex::command(builder); + tool.arg("markdown").arg(&output); + + let guard = + builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); + let _time = helpers::timeit(builder); + tool.run_capture(builder); + drop(guard); + // The tests themselves need to link to std, so make sure it is + // available. + builder.ensure(compile::Std::new(compiler, compiler.host)); + markdown_test(builder, compiler, &output); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version.rs new file mode 100644 index 00000000..ca559fed --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version.rs @@ -0,0 +1,4 @@ +fn get_browser_ui_test_version(builder: &Builder<'_>, npm: &Path) -> Option { + get_browser_ui_test_version_inner(builder, npm, false) + .or_else(|| get_browser_ui_test_version_inner(builder, npm, true)) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version_inner.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version_inner.rs new file mode 100644 index 00000000..31196cec --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/get_browser_ui_test_version_inner.rs @@ -0,0 +1,16 @@ +fn get_browser_ui_test_version_inner( + builder: &Builder<'_>, + npm: &Path, + global: bool, +) -> Option { + let mut command = command(npm); + command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); + if global { + command.arg("--global"); + } + let lines = command.allow_failure().run_capture(builder).stdout(); + lines + .lines() + .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) + .map(|v| v.to_owned()) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/html_check.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/html_check.rs new file mode 100644 index 00000000..30efa82a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/html_check.rs @@ -0,0 +1,43 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HtmlCheck { + target: TargetSelection, +} + +impl Step for HtmlCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.path("src/tools/html-checker"); + run.lazy_default_condition(Box::new(|| check_if_tidy_is_installed(builder))) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(HtmlCheck { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + if !check_if_tidy_is_installed(builder) { + eprintln!("not running HTML-check tool because `tidy` is missing"); + eprintln!( + "You need the HTML tidy tool https://www.html-tidy.org/, this tool is *not* part of the rust project and needs to be installed separately, for example via your package manager." + ); + panic!("Cannot run html-check tests"); + } + // Ensure that a few different kinds of documentation are available. + builder.default_doc(&[]); + builder.ensure(crate::core::build_steps::doc::Rustc::new( + builder.top_stage, + self.target, + builder, + )); + + builder + .tool_cmd(Tool::HtmlChecker) + .delay_failure() + .arg(builder.doc_out(self.target)) + .run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/linkcheck.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/linkcheck.rs new file mode 100644 index 00000000..801f5caf --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/linkcheck.rs @@ -0,0 +1,83 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Linkcheck { + host: TargetSelection, +} + +impl Step for Linkcheck { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let hosts = &builder.hosts; + let targets = &builder.targets; + + // if we have different hosts and targets, some things may be built for + // the host (e.g. rustc) and others for the target (e.g. std). The + // documentation built for each will contain broken links to + // docs built for the other platform (e.g. rustc linking to cargo) + if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { + panic!( + "Linkcheck currently does not support builds with different hosts and targets.\nYou can skip linkcheck with --skip src/tools/linkchecker" + ); + } + + builder.info(&format!("Linkcheck ({host})")); + + // Test the linkchecker itself. + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + Kind::Test, + "src/tools/linkchecker", + SourceType::InTree, + &[], + ); + run_cargo_test( + cargo, + &[], + &[], + "linkchecker", + "linkchecker self tests", + compiler, + bootstrap_host, + builder, + ); + + if builder.doc_tests == DocTests::No { + return; + } + + // Build all the default documentation. + builder.default_doc(&[]); + + // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. + let linkchecker = builder.tool_cmd(Tool::Linkchecker); + + // Run the linkchecker. + let _guard = + builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); + let _time = helpers::timeit(builder); + linkchecker.delay_failure().arg(builder.out.join(host).join("doc")).run(builder); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.path("src/tools/linkchecker"); + run.default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Linkcheck { host: run.target }); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/lint_docs.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/lint_docs.rs new file mode 100644 index 00000000..5888e53c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/lint_docs.rs @@ -0,0 +1,36 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LintDocs { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for LintDocs { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/lint-docs") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); + run.builder.ensure(LintDocs { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.config.build, + compiler, + target: run.target, + }, + }); + } + + /// Tests that the lint examples in the rustc book generate the correct + /// lints and have the expected format. + fn run(self, builder: &Builder<'_>) { + builder.ensure(crate::core::build_steps::doc::RustcBook { + compiler: self.common.compiler, + target: self.common.target, + validate: true, + }); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/markdown_test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/markdown_test.rs new file mode 100644 index 00000000..6606c392 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/markdown_test.rs @@ -0,0 +1,27 @@ +fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { + if let Ok(contents) = fs::read_to_string(markdown) { + if !contents.contains("```") { + return true; + } + } + + builder.verbose(|| println!("doc tests for: {}", markdown.display())); + let mut cmd = builder.rustdoc_cmd(compiler); + builder.add_rust_test_threads(&mut cmd); + // allow for unstable options such as new editions + cmd.arg("-Z"); + cmd.arg("unstable-options"); + cmd.arg("--test"); + cmd.arg(markdown); + cmd.env("RUSTC_BOOTSTRAP", "1"); + + let test_args = builder.config.test_args().join(" "); + cmd.arg("--test-args").arg(test_args); + + cmd = cmd.delay_failure(); + if !builder.config.verbose_tests { + cmd.run_capture(builder).is_success() + } else { + cmd.run(builder) + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/mir_opt.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/mir_opt.rs new file mode 100644 index 00000000..c4ba31bf --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/mir_opt.rs @@ -0,0 +1,63 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct MirOpt { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for MirOpt { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("tests/mir-opt") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(MirOpt { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + let run = |target| { + builder.ensure(Compiletest { + compiler: self.common.compiler, + target, + mode: "mir-opt", + suite: "mir-opt", + path: "tests/mir-opt", + compare_mode: None, + }) + }; + + run(self.common.target); + + // Run more targets with `--bless`. But we always run the host target first, since some + // tests use very specific `only` clauses that are not covered by the target set below. + if builder.config.cmd.bless() { + // All that we really need to do is cover all combinations of 32/64-bit and unwind/abort, + // but while we're at it we might as well flex our cross-compilation support. This + // selection covers all our tier 1 operating systems and architectures using only tier + // 1 targets. + + for target in ["aarch64-unknown-linux-gnu", "i686-pc-windows-msvc"] { + run(TargetSelection::from_user(target)); + } + + for target in ["x86_64-apple-darwin", "i686-unknown-linux-musl"] { + let target = TargetSelection::from_user(target); + let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { + compiler: self.common.compiler, + base: target, + }); + run(panic_abort_target); + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/miri.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/miri.rs new file mode 100644 index 00000000..fdd77866 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/miri.rs @@ -0,0 +1,172 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + pub common: common_test_fields::CommonTestFields, +} + +impl Miri { + /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. + pub fn build_miri_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + ) -> PathBuf { + let miri_sysroot = builder.out.join(compiler.host).join("miri-sysroot"); + let mut cargo = builder::Cargo::new( + builder, + compiler, + Mode::Std, + SourceType::Submodule, + target, + Kind::MiriSetup, + ); + + // Tell `cargo miri setup` where to find the sources. + cargo.env("MIRI_LIB_SRC", builder.src.join("library")); + // Tell it where to put the sysroot. + cargo.env("MIRI_SYSROOT", &miri_sysroot); + + let mut cargo = BootstrapCommand::from(cargo); + let _guard = + builder.msg(Kind::Build, compiler.stage, "miri sysroot", compiler.host, target); + cargo.run(builder); + + // # Determine where Miri put its sysroot. + // To this end, we run `cargo miri setup --print-sysroot` and capture the output. + // (We do this separately from the above so that when the setup actually + // happens we get some output.) + // We re-use the `cargo` from above. + cargo.arg("--print-sysroot"); + + builder.verbose(|| println!("running: {cargo:?}")); + let stdout = cargo.run_capture_stdout(builder).stdout(); + // Output is "\n". + let sysroot = stdout.trim_end(); + builder.verbose(|| println!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); + PathBuf::from(sysroot) + } +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Miri { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for miri. + fn run(self, builder: &Builder<'_>) { + let host = self.common.host; + let target = self.common.target; + let stage = self.common.stage; + if stage == 0 { + eprintln!("miri cannot be tested at stage 0"); + std::process::exit(1); + } + + // This compiler runs on the host, we'll just use it for the target. + let target_compiler = builder.compiler(stage, host); + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage + // compilers, which isn't what we want. Rustdoc should be linked in the same way as the + // rustc compiler it's paired with, so it must be built with the previous stage compiler. + let host_compiler = builder.compiler(stage - 1, host); + + // Build our tools. + let miri = builder.ensure(tool::Miri { + compiler: host_compiler, + target: host, + extra_features: Vec::new(), + }); + // the ui tests also assume cargo-miri has been built + builder.ensure(tool::CargoMiri { + compiler: host_compiler, + target: host, + extra_features: Vec::new(), + }); + + // We also need sysroots, for Miri and for the host (the latter for build scripts). + // This is for the tests so everything is done with the target compiler. + let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target); + builder.ensure(compile::Std::new(target_compiler, host)); + let host_sysroom = builder.sysroot(target_compiler); + + // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when + // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. + if !builder.config.dry_run { + let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); + // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see + // ). + // We can hence use that directly as a signal to clear the ui test dir. + builder.clear_if_dirty(&ui_test_dep_dir, &miri_sysroot); + } + + // Run `cargo test`. + // This is with the Miri crate, so it uses the host compiler. + let mut cargo = tool::prepare_tool_cargo( + builder, + host_compiler, + Mode::ToolRustc, + host, + Kind::Test, + "src/tools/miri", + SourceType::InTree, + &[], + ); + + cargo.add_rustc_lib_path(builder); + + // We can NOT use `run_cargo_test` since Miri's integration tests do not use the usual test + // harness and therefore do not understand the flags added by `add_flags_and_try_run_test`. + let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", host_compiler, host, builder); + + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", &miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", &host_sysroom); + cargo.env("MIRI", &miri); + + // Set the target. + cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); + + { + let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "miri", host, target); + let _time = helpers::timeit(builder); + cargo.run(builder); + } + + // Run it again for mir-opt-level 4 to catch some miscompilations. + if builder.config.test_args().is_empty() { + cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); + // Optimizations can change backtraces + cargo.env("MIRI_SKIP_UI_CHECKS", "1"); + // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible + cargo.env_remove("RUSTC_BLESS"); + // Optimizations can change error locations and remove UB so don't run `fail` tests. + cargo.args(["tests/pass", "tests/panic"]); + + { + let _guard = builder.msg_sysroot_tool( + Kind::Test, + stage, + "miri (mir-opt-level 4)", + host, + target, + ); + let _time = helpers::timeit(builder); + cargo.run(builder); + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/mod.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/mod.rs new file mode 100644 index 00000000..6155c99c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/mod.rs @@ -0,0 +1,48 @@ +pub mod book_test; +pub mod bootstrap; +pub mod cargo_miri; +pub mod cargo; +pub mod cargotest; +pub mod check_if_tidy_is_installed; +pub mod clippy; +pub mod codegen_cranelift; +pub mod codegen_gcc; +pub mod compiletest_test; +pub mod compiletest; +pub mod coverage; +pub mod crate_bootstrap; +pub mod crate_build_helper; +pub mod crate_librustc; +pub mod crate_run_make_support; +pub mod crate_rustdoc_json_types; +pub mod crate_rustdoc; +// pub mod crate_mod; +pub mod distcheck; +pub mod error_index; +pub mod get_browser_ui_test_version_inner; +pub mod get_browser_ui_test_version; +pub mod html_check; +pub mod linkcheck; +pub mod lint_docs; +pub mod markdown_test; +pub mod mir_opt; +pub mod miri; +pub mod path_for_cargo; +pub mod prepare_cargo_test; +pub mod remote_copy_libs; +pub mod run_cargo_test; +pub mod run_make_support; +pub mod run_make; +pub mod rust_analyzer; +pub mod rust_installer; +pub mod rustc_guide; +pub mod rustdoc_gui; +pub mod rustdoc_js_not_std; +pub mod rustdoc_js_std; +pub mod rustdoc_theme; +pub mod rustfmt; +pub mod test_float_parse; +pub mod test_helpers; +pub mod testdir; +pub mod tidy; +pub mod tier_check; diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/path_for_cargo.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/path_for_cargo.rs new file mode 100644 index 00000000..dea05002 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/path_for_cargo.rs @@ -0,0 +1,8 @@ +fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { + // Configure PATH to find the right rustc. NB. we have to use PATH + // and not RUSTC because the Cargo test suite has tests that will + // fail if rustc is not spelled `rustc`. + let path = builder.sysroot(compiler).join("bin"); + let old_path = env::var_os("PATH").unwrap_or_default(); + env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/prepare_cargo_test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/prepare_cargo_test.rs new file mode 100644 index 00000000..ca75ce10 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/prepare_cargo_test.rs @@ -0,0 +1,76 @@ +/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. +fn prepare_cargo_test( + cargo: impl Into, + libtest_args: &[&str], + crates: &[String], + primary_crate: &str, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> BootstrapCommand { + let mut cargo = cargo.into(); + + // Propagate `--bless` if it has not already been set/unset + // Any tools that want to use this should bless if `RUSTC_BLESS` is set to + // anything other than `0`. + if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { + cargo.env("RUSTC_BLESS", "Gesundheit"); + } + + // Pass in some standard flags then iterate over the graph we've discovered + // in `cargo metadata` with the maps above and figure out what `-p` + // arguments need to get passed. + if builder.kind == Kind::Test && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cargo.arg("--doc"); + } + DocTests::No => { + let krate = &builder + .crates + .get(primary_crate) + .unwrap_or_else(|| panic!("missing crate {primary_crate}")); + if krate.has_lib { + cargo.arg("--lib"); + } + cargo.args(["--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + for krate in crates { + cargo.arg("-p").arg(krate); + } + + cargo.arg("--").args(builder.config.test_args()).args(libtest_args); + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + // The tests are going to run with the *target* libraries, so we need to + // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. + // + // Note that to run the compiler we need to run with the *host* libraries, + // but our wrapper scripts arrange for that to be the case anyway. + // + // We skip everything on Miri as then this overwrites the libdir set up + // by `Cargo::new` and that actually makes things go wrong. + if builder.kind != Kind::Miri { + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*builder.sysroot_target_libdir(compiler, target))); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + } + + if builder.remote_tested(target) { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), + ); + } else if let Some(tool) = builder.runner(target) { + cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), tool); + } + + cargo +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/remote_copy_libs.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/remote_copy_libs.rs new file mode 100644 index 00000000..0f40ea1b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/remote_copy_libs.rs @@ -0,0 +1,44 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RemoteCopyLibs { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for RemoteCopyLibs { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + if !builder.remote_tested(target) { + return; + } + + builder.ensure(compile::Std::new(compiler, target)); + + builder.info(&format!("REMOTE copy libs to emulator ({target})")); + + let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + + // Spawn the emulator and wait for it to come online + let tool = builder.tool_exe(Tool::RemoteTestClient); + let mut cmd = command(&tool); + cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + if let Some(rootfs) = builder.qemu_rootfs(target) { + cmd.arg(rootfs); + } + cmd.run(builder); + + // Push all our dylibs to the emulator + for f in t!(builder.sysroot_target_libdir(compiler, target).read_dir()) { + let f = t!(f); + if helpers::is_dylib(&f.path()) { + command(&tool).arg("push").arg(f.path()).run(builder); + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/run_cargo_test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_cargo_test.rs new file mode 100644 index 00000000..535b5e29 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_cargo_test.rs @@ -0,0 +1,33 @@ +/// Given a `cargo test` subcommand, add the appropriate flags and run it. +/// +/// Returns whether the test succeeded. +#[allow(clippy::too_many_arguments)] // FIXME: reduce the number of args and remove this. +fn run_cargo_test<'a>( + cargo: impl Into, + libtest_args: &[&str], + crates: &[String], + primary_crate: &str, + description: impl Into>, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> bool { + let mut cargo = + prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); + let _time = helpers::timeit(builder); + let _group = description.into().and_then(|what| { + builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) + }); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: crates.iter().map(|c| c.to_string()).collect(), + target: target.triple.to_string(), + host: compiler.host.triple.to_string(), + stage: compiler.stage, + }, + builder, + ); + add_flags_and_try_run_tests(builder, &mut cargo) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make.rs new file mode 100644 index 00000000..c2d96074 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make.rs @@ -0,0 +1,45 @@ +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RunMake { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RunMake { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("tests/run-make") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.build_triple(), + }, + }); + run.builder.ensure(RunMake { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.common.compiler, + target: self.common.target, + mode: "run-make", + suite: "run-make", + path: "tests/run-make", + compare_mode: None, + }); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make_support.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make_support.rs new file mode 100644 index 00000000..1608317d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/run_make_support.rs @@ -0,0 +1,50 @@ +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct RunMakeSupport { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RunMakeSupport { + type Output = PathBuf; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.build_triple(), + }, + }); + } + + /// Builds run-make-support and returns the path to the resulting rlib. + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(compile::Std::new(self.common.compiler, self.common.target)); + + let cargo = tool::prepare_tool_cargo( + builder, + self.common.compiler, + Mode::ToolStd, + self.common.target, + Kind::Build, + "src/tools/run-make-support", + SourceType::InTree, + &[], + ); + + cargo.into_cmd().run(builder); + + let lib_name = "librun_make_support.rlib"; + let lib = builder.tools_dir(self.common.compiler).join(lib_name); + + let cargo_out = builder.cargo_out(self.common.compiler, Mode::ToolStd, self.common.target).join(lib_name); + builder.copy_link(&cargo_out, &lib); + lib + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_analyzer.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_analyzer.rs new file mode 100644 index 00000000..5308389a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_analyzer.rs @@ -0,0 +1,65 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-analyzer") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(Self { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for rust-analyzer + fn run(self, builder: &Builder<'_>) { + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; + + // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, + // but we do need the standard library to be present. + builder.ensure(compile::Rustc::new(compiler, host)); + + let workspace_path = "src/tools/rust-analyzer"; + // until the whole RA test suite runs on `i686`, we only run + // `proc-macro-srv` tests + let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + Kind::Test, + crate_path, + SourceType::InTree, + &["in-rust-tree".to_owned()], + ); + cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); + + let dir = builder.src.join(workspace_path); + // needed by rust-analyzer to find its own text fixtures, cf. + // https://github.com/rust-analyzer/expect-test/issues/33 + cargo.env("CARGO_WORKSPACE_DIR", &dir); + + // RA's test suite tries to write to the source directory, that can't + // work in Rust CI + cargo.env("SKIP_SLOW_TESTS", "1"); + + cargo.add_rustc_lib_path(builder); + run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_installer.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_installer.rs new file mode 100644 index 00000000..198a6083 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rust_installer.rs @@ -0,0 +1,59 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RustInstaller; + +impl Step for RustInstaller { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Ensure the version placeholder replacement tool builds + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + Kind::Test, + "src/tools/rust-installer", + SourceType::InTree, + &[], + ); + + let _guard = builder.msg( + Kind::Test, + compiler.stage, + "rust-installer", + bootstrap_host, + bootstrap_host, + ); + run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); + + // We currently don't support running the test.sh script outside linux(?) environments. + // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a + // set of scripts, which will likely allow dropping this if. + if bootstrap_host != "x86_64-unknown-linux-gnu" { + return; + } + + let mut cmd = command(builder.src.join("src/tools/rust-installer/test.sh")); + let tmpdir = testdir(builder, compiler.host).join("rust-installer"); + let _ = std::fs::remove_dir_all(&tmpdir); + let _ = std::fs::create_dir_all(&tmpdir); + cmd.current_dir(&tmpdir); + cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); + cmd.env("CARGO", &builder.initial_cargo); + cmd.env("RUSTC", &builder.initial_rustc); + cmd.env("TMP_DIR", &tmpdir); + cmd.delay_failure().run(builder); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-installer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustc_guide.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustc_guide.rs new file mode 100644 index 00000000..ae1d2a7c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustc_guide.rs @@ -0,0 +1,28 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RustcGuide; + +impl Step for RustcGuide { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/doc/rustc-dev-guide") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcGuide); + } + + fn run(self, builder: &Builder<'_>) { + let relative_path = "src/doc/rustc-dev-guide"; + builder.require_submodule(relative_path, None); + + let src = builder.src.join(relative_path); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook).delay_failure(); + rustbook_cmd.arg("linkcheck").arg(&src); + let toolstate = + if rustbook_cmd.run(builder) { ToolState::TestPass } else { ToolState::TestFail }; + builder.save_toolstate("rustc-dev-guide", toolstate); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_gui.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_gui.rs new file mode 100644 index 00000000..a514a34d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_gui.rs @@ -0,0 +1,99 @@ +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct RustdocGUI { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RustdocGUI { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.suite_path("tests/rustdoc-gui"); + run.lazy_default_condition(Box::new(move || { + builder.config.nodejs.is_some() + && builder.doc_tests != DocTests::Only + && builder + .config + .npm + .as_ref() + .map(|p| get_browser_ui_test_version(builder, p).is_some()) + .unwrap_or(false) + })) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocGUI { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.common.compiler, self.common.target)); + + let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); + + let out_dir = builder.test_out(self.common.target).join("rustdoc-gui"); + builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.common.compiler)); + + if let Some(src) = builder.config.src.to_str() { + cmd.arg("--rust-src").arg(src); + } + + if let Some(out_dir) = out_dir.to_str() { + cmd.arg("--out-dir").arg(out_dir); + } + + if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { + cmd.arg("--initial-cargo").arg(initial_cargo); + } + + cmd.arg("--jobs").arg(builder.jobs().to_string()); + + cmd.env("RUSTDOC", builder.rustdoc(self.common.compiler)) + .env("RUSTC", builder.rustc(self.common.compiler)); + + add_rustdoc_cargo_linker_args(&mut cmd, builder, self.common.compiler.host, LldThreads::No); + + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(p, "tests/rustdoc-gui", builder) { + if !p.ends_with(".goml") { + eprintln!("A non-goml file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-gui tests"); + } + if let Some(name) = path.file_name().and_then(|f| f.to_str()) { + cmd.arg("--goml-file").arg(name); + } + } + } + + for test_arg in builder.config.test_args() { + cmd.arg("--test-arg").arg(test_arg); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } + + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + + let _time = helpers::timeit(builder); + let _guard = builder.msg_sysroot_tool( + Kind::Test, + self.common.compiler.stage, + "rustdoc-gui", + self.common.compiler.host, + self.common.target, + ); + try_run_tests(builder, &mut cmd, true); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_not_std.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_not_std.rs new file mode 100644 index 00000000..bcadc650 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_not_std.rs @@ -0,0 +1,38 @@ +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSNotStd { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RustdocJSNotStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocJSNotStd { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.common.compiler, + target: self.common.target, + mode: "js-doc-test", + suite: "rustdoc-js", + path: "tests/rustdoc-js", + compare_mode: None, + }); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_std.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_std.rs new file mode 100644 index 00000000..ef72e9e8 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_js_std.rs @@ -0,0 +1,66 @@ +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSStd { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RustdocJSStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js-std").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(RustdocJSStd { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + let nodejs = + builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); + let mut command = command(nodejs); + command + .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) + .arg("--crate-name") + .arg("std") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--doc-folder") + .arg(builder.doc_out(self.common.target)) + .arg("--test-folder") + .arg(builder.src.join("tests/rustdoc-js-std")); + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) + { + if !p.ends_with(".js") { + eprintln!("A non-js file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-js-std tests"); + } + command.arg("--test-file").arg(path); + } + } + builder.ensure(crate::core::build_steps::doc::Std::new( + self.common.stage, + self.common.target, + DocumentationFormat::Html, + )); + let _guard = builder.msg( + Kind::Test, + self.common.stage, + "rustdoc-js-std", + builder.config.build, + self.common.target, + ); + command.run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_theme.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_theme.rs new file mode 100644 index 00000000..e870aa15 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustdoc_theme.rs @@ -0,0 +1,43 @@ +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct RustdocTheme { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for RustdocTheme { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc-themes") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + + run.builder.ensure(RustdocTheme { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + fn run(self, builder: &Builder<'_>) { + let rustdoc = builder.bootstrap_out.join("rustdoc"); + let mut cmd = builder.tool_cmd(Tool::RustdocTheme); + cmd.arg(rustdoc.to_str().unwrap()) + .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) + .env("RUSTC_STAGE", self.common.compiler.stage.to_string()) + .env("RUSTC_SYSROOT", builder.sysroot(self.common.compiler)) + .env("RUSTDOC_LIBDIR", builder.sysroot_target_libdir(self.common.compiler, self.common.compiler.host)) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("RUSTDOC_REAL", builder.rustdoc(self.common.compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + cmd.args(linker_args(builder, self.common.compiler.host, LldThreads::No)); + + cmd.delay_failure().run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/rustfmt.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustfmt.rs new file mode 100644 index 00000000..955ace5e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/rustfmt.rs @@ -0,0 +1,53 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Rustfmt { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for Rustfmt { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustfmt") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Rustfmt { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Runs `cargo test` for rustfmt. + fn run(self, builder: &Builder<'_>) { + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; + + builder.ensure(tool::Rustfmt { compiler, target: self.common.host, extra_features: Vec::new() }); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + Kind::Test, + "src/tools/rustfmt", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + cargo.env("RUSTFMT_TEST_DIR", dir); + + cargo.add_rustc_lib_path(builder); + + run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/test_float_parse.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/test_float_parse.rs new file mode 100644 index 00000000..c2379d9c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/test_float_parse.rs @@ -0,0 +1,91 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TestFloatParse { + pub common: common_test_fields::CommonTestFields, + path: PathBuf, +} + +impl Step for TestFloatParse { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/etc/test-float-parse") + } + + fn make_run(run: RunConfig<'_>) { + for path in run.paths { + let path = path.assert_single_path().path.clone(); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Self { + path, + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = self.common.host; + let compiler = self.common.compiler; + let path = self.path.to_str().unwrap(); + let crate_name = self.path + .components() + .last() + .unwrap() + .as_os_str() + .to_str() + .unwrap(); + + builder.ensure(tool::TestFloatParse { host: self.common.host }); + + // Run any unit tests in the crate + let cargo_test = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + bootstrap_host, + Kind::Test, + path, + SourceType::InTree, + &[], + ); + + run_cargo_test( + cargo_test, + &[], + &[], + crate_name, + crate_name, + compiler, + bootstrap_host, + builder, + ); + + // Run the actual parse tests. + let mut cargo_run = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + bootstrap_host, + Kind::Run, + path, + SourceType::InTree, + &[], + ); + + cargo_run.arg("--"); + if builder.config.args().is_empty() { + // By default, exclude tests that take longer than ~1m. + cargo_run.arg("--skip-huge"); + } else { + cargo_run.args(builder.config.args()); + } + + cargo_run.into_cmd().run(builder); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/test_helpers.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/test_helpers.rs new file mode 100644 index 00000000..5b25503d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/test_helpers.rs @@ -0,0 +1,68 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TestHelpers { + pub common: common_test_fields::CommonTestFields, +} + +impl Step for TestHelpers { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("tests/auxiliary/rust_test_helpers.c") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(TestHelpers { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); + } + + /// Compiles the `rust_test_helpers.c` library which we used in various + /// `run-pass` tests for ABI testing. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run { + return; + } + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.common.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.common.target + }; + let dst = builder.test_helpers_out(target); + let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); + if up_to_date(&src, &dst.join("librust_test_helpers.a")) { + return; + } + + let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); + t!(fs::create_dir_all(&dst)); + let mut cfg = cc::Build::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform cc of these compilers. Note, though, that + // on MSVC we still need cc's detection of env vars (ugh). + if !target.is_msvc() { + if let Some(ar) = builder.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(target)); + } + cfg.cargo_metadata(false) + .out_dir(&dst) + .target(&target.triple) + .host(&builder.config.build.triple) + .opt_level(0) + .warnings(false) + .debug(false) + .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) + .compile("rust_test_helpers"); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/testdir.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/testdir.rs new file mode 100644 index 00000000..6e1c5c70 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/testdir.rs @@ -0,0 +1,3 @@ +fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { + builder.out.join(host).join("test") +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/tidy.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/tidy.rs new file mode 100644 index 00000000..0a6de192 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/tidy.rs @@ -0,0 +1,86 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Tidy; + +impl Step for Tidy { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `tidy` tool. + /// + /// This tool in `src/tools` checks up on various bits and pieces of style and + /// otherwise just implements a few lint-like checks that are specific to the + /// compiler itself. + /// + /// Once tidy passes, this step also runs `fmt --check` if tests are being run + /// for the `dev` or `nightly` channels. + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::Tidy); + cmd.arg(&builder.src); + cmd.arg(&builder.initial_cargo); + cmd.arg(&builder.out); + // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. + let jobs = builder.config.jobs.unwrap_or_else(|| { + 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }); + cmd.arg(jobs.to_string()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + if let Some(s) = builder.config.cmd.extra_checks() { + cmd.arg(format!("--extra-checks={s}")); + } + let mut args = std::env::args_os(); + if args.any(|arg| arg == OsStr::new("--")) { + cmd.arg("--"); + cmd.args(args); + } + + if builder.config.channel == "dev" || builder.config.channel == "nightly" { + builder.info("fmt check"); + if builder.initial_rustfmt().is_none() { + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); + eprintln!( + r#"ERROR: no `rustfmt` binary found in {PATH}\nINFO: `rust.channel` is currently set to "{CHAN}"\nHELP: if you are testing a beta branch, set `rust.channel` to "beta" in the `config.toml` file\nHELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`,"#, + PATH = inferred_rustfmt_dir.display(), + CHAN = builder.config.channel, + ); + crate::exit!(1); + } + let all = false; + crate::core::build_steps::format::format(builder, !builder.config.cmd.bless(), all, &[ + ]); + } + + builder.info("tidy check"); + cmd.delay_failure().run(builder); + + builder.info("x.py completions check"); + let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] + .map(|filename| builder.src.join("src/etc/completions").join(filename)); + if builder.config.cmd.bless() { + builder.ensure(crate::core::build_steps::run::GenerateCompletions); + } else if get_completion(shells::Bash, &bash).is_some() + || get_completion(shells::Fish, &fish).is_some() + || get_completion(shells::PowerShell, &powershell).is_some() + || crate::flags::get_completion(shells::Zsh, &zsh).is_some() + { + eprintln!( + "x.py completions were changed; run `x.py run generate-completions` to update them" + ); + crate::exit!(1); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.doc_tests != DocTests::Only; + run.path("src/tools/tidy").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Tidy); + } +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_split/tier_check.rs b/standalonex/src/bootstrap/src/core/build_steps/test_split/tier_check.rs new file mode 100644 index 00000000..378fa909 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_split/tier_check.rs @@ -0,0 +1,49 @@ +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TierCheck { + pub compiler: Compiler, +} + +impl Step for TierCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tier-check") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(TierCheck { compiler }); + } + + /// Tests the Platform Support page in the rustc book. + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + self.compiler, + Mode::ToolStd, + self.compiler.host, + Kind::Run, + "src/tools/tier-check", + SourceType::InTree, + &[], + ); + cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); + cargo.arg(builder.rustc(self.compiler)); + if builder.is_verbose() { + cargo.arg("--verbose"); + } + + let _guard = builder.msg( + Kind::Test, + self.compiler.stage, + "platform support check", + self.compiler.host, + self.compiler.host, + ); + BootstrapCommand::from(cargo).delay_failure().run(builder); + } +} diff --git a/standalonex/src/src/core/build_steps/test.rs b/standalonex/src/bootstrap/src/core/build_steps/test_temp.rs similarity index 89% rename from standalonex/src/src/core/build_steps/test.rs rename to standalonex/src/bootstrap/src/core/build_steps/test_temp.rs index dcea9f5f..dc4d827f 100644 --- a/standalonex/src/src/core/build_steps/test.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/test_temp.rs @@ -1,7 +1,8 @@ -//! Build-and-run steps for `./x.py test` test fixtures -//! -//! `./x.py test` (aka [`Kind::Test`]) is currently allowed to reach build steps in other modules. -//! However, this contains ~all test parts we expect people to be able to build and run locally. +use crate::prelude::*; +// Build-and-run steps for `./x.py test` test fixtures +// +// `./x.py test` (aka [`Kind::Test`]) is currently allowed to reach build steps in other modules. +// However, this contains ~all test parts we expect people to be able to build and run locally. use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; @@ -18,7 +19,8 @@ use crate::core::builder::{ self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, crate_description, }; use crate::core::config::TargetSelection; -use crate::core::config::flags::{Subcommand, get_completion}; +use crate::core::config::flags::get_completion; +use crate::Subcommand; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, LldThreads, add_link_lib_path, add_rustdoc_cargo_linker_args, dylib_path, dylib_path_var, @@ -27,6 +29,8 @@ use crate::utils::helpers::{ use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; use crate::{CLang, DocTests, GitRepo, Mode, envify}; +mod common_test_fields; + const ADB_TEST_DIR: &str = "/data/local/tmp/work"; #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -212,8 +216,7 @@ impl Step for HtmlCheck { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Cargotest { - stage: u32, - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for Cargotest { @@ -225,7 +228,15 @@ impl Step for Cargotest { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Cargotest { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. @@ -233,7 +244,8 @@ impl Step for Cargotest { /// This tool in `src/tools` will check out a few Rust projects and run `cargo /// test` to ensure that we don't regress the test suites there. fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); + let compiler = self.common.compiler; + let host = self.common.host; builder.ensure(compile::Rustc::new(compiler, compiler.host)); let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); @@ -257,8 +269,7 @@ impl Step for Cargotest { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Cargo { - stage: u32, - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for Cargo { @@ -270,19 +281,28 @@ impl Step for Cargo { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Cargo { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for `cargo` packaged with Rust. fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); + let compiler = self.common.compiler; + let host = self.common.host; - builder.ensure(tool::Cargo { compiler, target: self.host }); + builder.ensure(tool::Cargo { compiler, target: self.common.host }); let cargo = tool::prepare_tool_cargo( builder, compiler, Mode::ToolRustc, - self.host, + self.common.host, Kind::Test, "src/tools/cargo", SourceType::Submodule, @@ -290,7 +310,7 @@ impl Step for Cargo { ); // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` - let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); + let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.common.host, builder); // Don't run cross-compile tests, we may not have cross-compiled libstd libs // available. @@ -304,9 +324,9 @@ impl Step for Cargo { builder.metrics.begin_test_suite( build_helper::metrics::TestSuiteMetadata::CargoPackage { crates: vec!["cargo".into()], - target: self.host.triple.to_string(), - host: self.host.triple.to_string(), - stage: self.stage, + target: self.common.host.triple.to_string(), + host: self.common.host.triple.to_string(), + stage: self.common.stage, }, builder, ); @@ -318,8 +338,7 @@ impl Step for Cargo { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct RustAnalyzer { - stage: u32, - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for RustAnalyzer { @@ -332,14 +351,22 @@ impl Step for RustAnalyzer { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(Self { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for rust-analyzer fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, // but we do need the standard library to be present. @@ -377,8 +404,7 @@ impl Step for RustAnalyzer { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Rustfmt { - stage: u32, - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for Rustfmt { @@ -390,16 +416,24 @@ impl Step for Rustfmt { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Rustfmt { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for rustfmt. fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; - builder.ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }); + builder.ensure(tool::Rustfmt { compiler, target: self.common.host, extra_features: Vec::new() }); let mut cargo = tool::prepare_tool_cargo( builder, @@ -424,7 +458,7 @@ impl Step for Rustfmt { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Miri { - target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Miri { @@ -479,14 +513,22 @@ impl Step for Miri { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { target: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Miri { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for miri. fn run(self, builder: &Builder<'_>) { - let host = builder.build.build; - let target = self.target; - let stage = builder.top_stage; + let host = self.common.host; + let target = self.common.target; + let stage = self.common.stage; if stage == 0 { eprintln!("miri cannot be tested at stage 0"); std::process::exit(1); @@ -521,7 +563,7 @@ impl Step for Miri { // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. - if !builder.config.dry_run() { + if !builder.config.dry_run { let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see // ). @@ -589,7 +631,7 @@ impl Step for Miri { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CargoMiri { - target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CargoMiri { @@ -601,14 +643,22 @@ impl Step for CargoMiri { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CargoMiri { target: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CargoMiri { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); } /// Tests `cargo miri test`. fn run(self, builder: &Builder<'_>) { - let host = builder.build.build; - let target = self.target; - let stage = builder.top_stage; + let host = self.common.host; + let target = self.common.target; + let stage = self.common.stage; if stage == 0 { eprintln!("cargo-miri cannot be tested at stage 0"); std::process::exit(1); @@ -657,7 +707,7 @@ impl Step for CargoMiri { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CompiletestTest { - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CompiletestTest { @@ -668,13 +718,21 @@ impl Step for CompiletestTest { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CompiletestTest { host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CompiletestTest { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for compiletest. fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(builder.top_stage, host); + let host = self.common.host; + let compiler = self.common.compiler; // We need `ToolStd` for the locally-built sysroot because // compiletest uses unstable features of the `test` crate. @@ -707,8 +765,7 @@ impl Step for CompiletestTest { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Clippy { - stage: u32, - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for Clippy { @@ -721,16 +778,24 @@ impl Step for Clippy { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Clippy { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for clippy. fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); + let stage = self.common.stage; + let host = self.common.host; + let compiler = self.common.compiler; - builder.ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }); + builder.ensure(tool::Clippy { compiler, target: self.common.host, extra_features: Vec::new() }); let mut cargo = tool::prepare_tool_cargo( builder, compiler, @@ -775,7 +840,7 @@ fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustdocTheme { - pub compiler: Compiler, + pub common: common_test_fields::CommonTestFields, } impl Step for RustdocTheme { @@ -790,7 +855,14 @@ impl Step for RustdocTheme { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.target); - run.builder.ensure(RustdocTheme { compiler }); + run.builder.ensure(RustdocTheme { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { @@ -798,13 +870,13 @@ impl Step for RustdocTheme { let mut cmd = builder.tool_cmd(Tool::RustdocTheme); cmd.arg(rustdoc.to_str().unwrap()) .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) - .env("RUSTC_STAGE", self.compiler.stage.to_string()) - .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) - .env("RUSTDOC_LIBDIR", builder.sysroot_target_libdir(self.compiler, self.compiler.host)) + .env("RUSTC_STAGE", self.common.compiler.stage.to_string()) + .env("RUSTC_SYSROOT", builder.sysroot(self.common.compiler)) + .env("RUSTDOC_LIBDIR", builder.sysroot_target_libdir(self.common.compiler, self.common.compiler.host)) .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) + .env("RUSTDOC_REAL", builder.rustdoc(self.common.compiler)) .env("RUSTC_BOOTSTRAP", "1"); - cmd.args(linker_args(builder, self.compiler.host, LldThreads::No)); + cmd.args(linker_args(builder, self.common.compiler.host, LldThreads::No)); cmd.delay_failure().run(builder); } @@ -812,7 +884,7 @@ impl Step for RustdocTheme { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustdocJSStd { - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for RustdocJSStd { @@ -826,7 +898,15 @@ impl Step for RustdocJSStd { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustdocJSStd { target: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(RustdocJSStd { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { @@ -840,7 +920,7 @@ impl Step for RustdocJSStd { .arg("--resource-suffix") .arg(&builder.version) .arg("--doc-folder") - .arg(builder.doc_out(self.target)) + .arg(builder.doc_out(self.common.target)) .arg("--test-folder") .arg(builder.src.join("tests/rustdoc-js-std")); for path in &builder.paths { @@ -854,16 +934,16 @@ impl Step for RustdocJSStd { } } builder.ensure(crate::core::build_steps::doc::Std::new( - builder.top_stage, - self.target, + self.common.stage, + self.common.target, DocumentationFormat::Html, )); let _guard = builder.msg( Kind::Test, - builder.top_stage, + self.common.stage, "rustdoc-js-std", builder.config.build, - self.target, + self.common.target, ); command.run(builder); } @@ -871,8 +951,7 @@ impl Step for RustdocJSStd { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustdocJSNotStd { - pub target: TargetSelection, - pub compiler: Compiler, + pub common: common_test_fields::CommonTestFields, } impl Step for RustdocJSNotStd { @@ -887,13 +966,20 @@ impl Step for RustdocJSNotStd { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); + run.builder.ensure(RustdocJSNotStd { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, + compiler: self.common.compiler, + target: self.common.target, mode: "js-doc-test", suite: "rustdoc-js", path: "tests/rustdoc-js", @@ -926,8 +1012,7 @@ fn get_browser_ui_test_version(builder: &Builder<'_>, npm: &Path) -> Option) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocGUI { target: run.target, compiler }); + run.builder.ensure(RustdocGUI { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); + builder.ensure(compile::Std::new(self.common.compiler, self.common.target)); let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); - let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); + let out_dir = builder.test_out(self.common.target).join("rustdoc-gui"); + builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.common.compiler)); if let Some(src) = builder.config.src.to_str() { cmd.arg("--rust-src").arg(src); @@ -977,13 +1069,13 @@ impl Step for RustdocGUI { cmd.arg("--jobs").arg(builder.jobs().to_string()); - cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) - .env("RUSTC", builder.rustc(self.compiler)); + cmd.env("RUSTDOC", builder.rustdoc(self.common.compiler)) + .env("RUSTC", builder.rustc(self.common.compiler)); - add_rustdoc_cargo_linker_args(&mut cmd, builder, self.compiler.host, LldThreads::No); + add_rustdoc_cargo_linker_args(&mut cmd, builder, self.common.compiler.host, LldThreads::No); for path in &builder.paths { - if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { + if let Some(p) = helpers::is_valid_test_suite_arg(p, "tests/rustdoc-gui", builder) { if !p.ends_with(".goml") { eprintln!("A non-goml file was given: `{}`", path.display()); panic!("Cannot run rustdoc-gui tests"); @@ -1009,10 +1101,10 @@ impl Step for RustdocGUI { let _time = helpers::timeit(builder); let _guard = builder.msg_sysroot_tool( Kind::Test, - self.compiler.stage, + self.common.compiler.stage, "rustdoc-gui", - self.compiler.host, - self.target, + self.common.compiler.host, + self.common.target, ); try_run_tests(builder, &mut cmd, true); } @@ -1255,8 +1347,7 @@ macro_rules! coverage_test_alias { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] pub struct RunMakeSupport { - pub compiler: Compiler, - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for RunMakeSupport { @@ -1269,18 +1360,25 @@ impl Step for RunMakeSupport { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); + run.builder.ensure(RunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.build_triple(), + }, + }); } /// Builds run-make-support and returns the path to the resulting rlib. fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(compile::Std::new(self.compiler, self.target)); + builder.ensure(compile::Std::new(self.common.compiler, self.common.target)); let cargo = tool::prepare_tool_cargo( builder, - self.compiler, + self.common.compiler, Mode::ToolStd, - self.target, + self.common.target, Kind::Build, "src/tools/run-make-support", SourceType::InTree, @@ -1290,9 +1388,9 @@ impl Step for RunMakeSupport { cargo.into_cmd().run(builder); let lib_name = "librun_make_support.rlib"; - let lib = builder.tools_dir(self.compiler).join(lib_name); + let lib = builder.tools_dir(self.common.compiler).join(lib_name); - let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(lib_name); + let cargo_out = builder.cargo_out(self.common.compiler, Mode::ToolStd, self.common.target).join(lib_name); builder.copy_link(&cargo_out, &lib); lib } @@ -1300,7 +1398,7 @@ impl Step for RunMakeSupport { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CrateRunMakeSupport { - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CrateRunMakeSupport { @@ -1312,13 +1410,21 @@ impl Step for CrateRunMakeSupport { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrateRunMakeSupport { host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CrateRunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for run-make-support. fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(0, host); + let host = self.common.host; + let compiler = self.common.compiler; let mut cargo = tool::prepare_tool_cargo( builder, @@ -1346,7 +1452,7 @@ impl Step for CrateRunMakeSupport { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CrateBuildHelper { - host: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CrateBuildHelper { @@ -1358,13 +1464,21 @@ impl Step for CrateBuildHelper { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrateBuildHelper { host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(CrateBuildHelper { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Runs `cargo test` for build_helper. fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(0, host); + let host = self.common.host; + let compiler = self.common.compiler; let mut cargo = tool::prepare_tool_cargo( builder, @@ -1424,8 +1538,7 @@ host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); /// tests. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct RunMake { - pub compiler: Compiler, - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for RunMake { @@ -1439,14 +1552,28 @@ impl Step for RunMake { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - run.builder.ensure(RunMake { compiler, target: run.target }); + run.builder.ensure(RunMakeSupport { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.build_triple(), + }, + }); + run.builder.ensure(RunMake { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, + compiler: self.common.compiler, + target: self.common.target, mode: "run-make", suite: "run-make", path: "tests/run-make", @@ -1473,8 +1600,7 @@ default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assem /// just that mode.) #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Coverage { - pub compiler: Compiler, - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Coverage { @@ -1522,13 +1648,20 @@ impl Step for Coverage { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(Coverage { compiler, target: run.target }); + run.builder.ensure(Coverage { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { // Run the specified coverage tests (possibly all of them) in both modes. - Self::run_coverage_tests(builder, self.compiler, self.target, CoverageMap::MODE); - Self::run_coverage_tests(builder, self.compiler, self.target, CoverageRun::MODE); + Self::run_coverage_tests(builder, self.common.compiler, self.common.target, CoverageMap::MODE); + Self::run_coverage_tests(builder, self.common.compiler, self.common.target, CoverageRun::MODE); } } @@ -1555,11 +1688,9 @@ host_test!(CoverageRunRustdoc { suite: "coverage-run-rustdoc" }); -// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MirOpt { - pub compiler: Compiler, - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for MirOpt { @@ -1573,13 +1704,20 @@ impl Step for MirOpt { fn make_run(run: RunConfig<'_>) { let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(MirOpt { compiler, target: run.target }); + run.builder.ensure(MirOpt { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { let run = |target| { builder.ensure(Compiletest { - compiler: self.compiler, + compiler: self.common.compiler, target, mode: "mir-opt", suite: "mir-opt", @@ -1588,7 +1726,7 @@ impl Step for MirOpt { }) }; - run(self.target); + run(self.common.target); // Run more targets with `--bless`. But we always run the host target first, since some // tests use very specific `only` clauses that are not covered by the target set below. @@ -1605,7 +1743,7 @@ impl Step for MirOpt { for target in ["x86_64-apple-darwin", "i686-unknown-linux-musl"] { let target = TargetSelection::from_user(target); let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { - compiler: self.compiler, + compiler: self.common.compiler, base: target, }); run(panic_abort_target); @@ -1951,7 +2089,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the if builder.config.llvm_enabled(compiler.host) { let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target: builder.config.build }); - if !builder.config.dry_run() { + if !builder.config.dry_run { let llvm_version = command(&llvm_config).arg("--version").run_capture_stdout(builder).stdout(); let llvm_components = @@ -1973,13 +2111,13 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // requirement, but the `-L` library path is not propagated across // separate compilations. We can add LLVM's library path to the // platform-specific environment variable as a workaround. - if !builder.config.dry_run() && suite.ends_with("fulldeps") { + if !builder.config.dry_run && suite.ends_with("fulldeps") { let llvm_libdir = command(&llvm_config).arg("--libdir").run_capture_stdout(builder).stdout(); add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); } - if !builder.config.dry_run() && matches!(mode, "run-make" | "coverage-run") { + if !builder.config.dry_run && matches!(mode, "run-make" | "coverage-run") { // The llvm/bin directory contains many useful cross-platform // tools. Pass the path to run-make tests so they can use them. // (The coverage-run tests also need these tools to process @@ -1991,7 +2129,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); } - if !builder.config.dry_run() && mode == "run-make" { + if !builder.config.dry_run && mode == "run-make" { // If LLD is available, add it to the PATH if builder.config.lld_enabled { let lld_install_root = @@ -2011,7 +2149,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // Only pass correct values for these flags for the `run-make` suite as it // requires that a C++ compiler was configured which isn't always the case. - if !builder.config.dry_run() && mode == "run-make" { + if !builder.config.dry_run && mode == "run-make" { cmd.arg("--cc") .arg(builder.cc(target)) .arg("--cxx") @@ -2052,7 +2190,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // // Note that if we encounter `PATH` we make sure to append to our own `PATH` // rather than stomp over it. - if !builder.config.dry_run() && target.is_msvc() { + if !builder.config.dry_run && target.is_msvc() { for (k, v) in builder.cc.borrow()[&target].env() { if k != "PATH" { cmd.env(k, v); @@ -2062,7 +2200,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the } // Special setup to enable running with sanitizers on MSVC. - if !builder.config.dry_run() + if !builder.config.dry_run && target.contains("msvc") && builder.config.sanitizers_enabled(target) { @@ -2107,7 +2245,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--adb-path").arg("adb"); cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); - if target.contains("android") && !builder.config.dry_run() { + if target.contains("android") && !builder.config.dry_run { // Assume that cc for this target comes from the android sysroot cmd.arg("--android-cross-path") .arg(builder.cc(target).parent().unwrap().parent().unwrap()); @@ -2350,7 +2488,7 @@ test_book!( #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ErrorIndex { - compiler: Compiler, + pub common: common_test_fields::CommonTestFields, } impl Step for ErrorIndex { @@ -2366,8 +2504,16 @@ impl Step for ErrorIndex { // error_index_generator depends on librustdoc. Use the compiler that // is normally used to build rustdoc for other tests (like compiletest // tests in tests/rustdoc) so that it shares the same artifacts. - let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(ErrorIndex { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.build.build, + compiler, + target: run.target, + }, + }); } /// Runs the error index generator tool to execute the tests located in the error @@ -2377,7 +2523,7 @@ impl Step for ErrorIndex { /// generate a markdown file from the error indexes of the code base which is /// then passed to `rustdoc --test`. fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; + let compiler = self.common.compiler; let dir = testdir(builder, compiler.host); t!(fs::create_dir_all(&dir)); @@ -2538,7 +2684,7 @@ fn prepare_cargo_test( ) -> BootstrapCommand { let mut cargo = cargo.into(); - // Propegate `--bless` if it has not already been set/unset + // Propagate `--bless` if it has not already been set/unset // Any tools that want to use this should bless if `RUSTC_BLESS` is set to // anything other than `0`. if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { @@ -3133,8 +3279,7 @@ impl Step for TierCheck { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct LintDocs { - pub compiler: Compiler, - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for LintDocs { @@ -3147,9 +3292,14 @@ impl Step for LintDocs { } fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); run.builder.ensure(LintDocs { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.builder.config.build, + compiler, + target: run.target, + }, }); } @@ -3157,8 +3307,8 @@ impl Step for LintDocs { /// lints and have the expected format. fn run(self, builder: &Builder<'_>) { builder.ensure(crate::core::build_steps::doc::RustcBook { - compiler: self.compiler, - target: self.target, + compiler: self.common.compiler, + target: self.common.target, validate: true, }); } @@ -3226,7 +3376,7 @@ impl Step for RustInstaller { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct TestHelpers { - pub target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for TestHelpers { @@ -3237,22 +3387,30 @@ impl Step for TestHelpers { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TestHelpers { target: run.target }) + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(TestHelpers { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } /// Compiles the `rust_test_helpers.c` library which we used in various /// `run-pass` tests for ABI testing. fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { + if builder.config.dry_run { return; } // The x86_64-fortanix-unknown-sgx target doesn't have a working C // toolchain. However, some x86_64 ELF objects can be linked // without issues. Use this hack to compile the test helpers. - let target = if self.target == "x86_64-fortanix-unknown-sgx" { + let target = if self.common.target == "x86_64-fortanix-unknown-sgx" { TargetSelection::from_user("x86_64-unknown-linux-gnu") } else { - self.target + self.common.target }; let dst = builder.test_helpers_out(target); let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); @@ -3287,8 +3445,7 @@ impl Step for TestHelpers { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CodegenCranelift { - compiler: Compiler, - target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CodegenCranelift { @@ -3329,12 +3486,19 @@ impl Step for CodegenCranelift { return; } - builder.ensure(CodegenCranelift { compiler, target: run.target }); + builder.ensure(CodegenCranelift { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; + let compiler = self.common.compiler; + let target = self.common.target; builder.ensure(compile::Std::new(compiler, target)); @@ -3412,8 +3576,7 @@ impl Step for CodegenCranelift { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CodegenGCC { - compiler: Compiler, - target: TargetSelection, + pub common: common_test_fields::CommonTestFields, } impl Step for CodegenGCC { @@ -3457,12 +3620,19 @@ impl Step for CodegenGCC { return; } - builder.ensure(CodegenGCC { compiler, target: run.target }); + builder.ensure(CodegenGCC { + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.build_triple(), + compiler, + target: run.target, + }, + }); } fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; + let compiler = self.common.compiler; + let target = self.common.target; builder.ensure(compile::Std::new_with_extra_rust_args(compiler, target, &[ "-Csymbol-mangling-version=v0", @@ -3536,8 +3706,8 @@ impl Step for CodegenGCC { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct TestFloatParse { + pub common: common_test_fields::CommonTestFields, path: PathBuf, - host: TargetSelection, } impl Step for TestFloatParse { @@ -3552,17 +3722,32 @@ impl Step for TestFloatParse { fn make_run(run: RunConfig<'_>) { for path in run.paths { let path = path.assert_single_path().path.clone(); - run.builder.ensure(Self { path, host: run.target }); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + run.builder.ensure(Self { + path, + common: common_test_fields::CommonTestFields { + stage: run.builder.top_stage, + host: run.target, + compiler, + target: run.target, + }, + }); } } fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(builder.top_stage, bootstrap_host); + let bootstrap_host = self.common.host; + let compiler = self.common.compiler; let path = self.path.to_str().unwrap(); - let crate_name = self.path.components().last().unwrap().as_os_str().to_str().unwrap(); - - builder.ensure(tool::TestFloatParse { host: self.host }); + let crate_name = self.path + .components() + .last() + .unwrap() + .as_os_str() + .to_str() + .unwrap(); + + builder.ensure(tool::TestFloatParse { host: self.common.host }); // Run any unit tests in the crate let cargo_test = tool::prepare_tool_cargo( diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_utils/Cargo.toml b/standalonex/src/bootstrap/src/core/build_steps/test_utils/Cargo.toml new file mode 100644 index 00000000..31b0e6e9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_utils/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bootstrap-test-utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +# Add necessary dependencies here diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/check_if_tidy_is_installed.rs b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/check_if_tidy_is_installed.rs new file mode 100644 index 00000000..dfb46d8a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/check_if_tidy_is_installed.rs @@ -0,0 +1,6 @@ +use bootstrap::Builder; + +pub fn check_if_tidy_is_installed(builder: &Builder<'_>) -> bool { + builder.config.tidy + && builder.config.tools.as_ref().map_or(false, |tools| tools.contains("tidy")) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/get_browser_ui_test_version.rs b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/get_browser_ui_test_version.rs new file mode 100644 index 00000000..8cb912f2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/get_browser_ui_test_version.rs @@ -0,0 +1,16 @@ +use bootstrap::Builder; +use std::path::Path; + +pub fn get_browser_ui_test_version(builder: &Builder<'_>, npm: &Path) -> Option { + builder.info.browser_ui_test + .as_ref() + .and_then(|s| { + if s == "auto" { + // Assuming get_browser_ui_test_version_inner will also be moved + // For now, I'll keep it as is and fix it later. + get_browser_ui_test_version_inner(builder, npm) + } else { + Some(s.to_string()) + } + }) +} diff --git a/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/lib.rs b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/lib.rs new file mode 100644 index 00000000..25c8c68a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/build_steps/test_utils/src/lib.rs @@ -0,0 +1 @@ +// This will be the lib.rs for the new bootstrap-test-utils crate diff --git a/standalonex/src/bootstrap/src/core/build_steps/toolstate.rs b/standalonex/src/bootstrap/src/core/build_steps/toolstate.rs index 8ac311b2..3a2aa111 100644 --- a/standalonex/src/bootstrap/src/core/build_steps/toolstate.rs +++ b/standalonex/src/bootstrap/src/core/build_steps/toolstate.rs @@ -150,7 +150,7 @@ impl Step for ToolStateCheck { /// stable tool. That is, the status is not allowed to get worse /// (test-pass to test-fail or build-fail). fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -261,7 +261,7 @@ impl Builder<'_> { // If we're in a dry run setting we don't want to save toolstates as // that means if we e.g. panic down the line it'll look like we tested // everything (but we actually haven't). - if self.config.dry_run() { + if self.config.dry_run { return; } // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking diff --git a/standalonex/src/bootstrap/src/core/builder/cargo.rs b/standalonex/src/bootstrap/src/core/builder/cargo.rs index 0688a1d6..e07d5bb4 100644 --- a/standalonex/src/bootstrap/src/core/builder/cargo.rs +++ b/standalonex/src/bootstrap/src/core/builder/cargo.rs @@ -1,3 +1,4 @@ +use crate::prelude::*; use std::env; use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; @@ -264,7 +265,7 @@ impl Cargo { self.rustdocflags.arg(&arg); } - if !builder.config.dry_run() + if !builder.config.dry_run && builder.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") { self.rustflags.arg("-Clink-arg=-gz"); @@ -759,7 +760,7 @@ impl Builder<'_> { // // Only clear out the directory if we're compiling std; otherwise, we // should let Cargo take care of things for us (via depdep info) - if !self.config.dry_run() && mode == Mode::Std && cmd_kind == Kind::Build { + if !self.config.dry_run && mode == Mode::Std && cmd_kind == Kind::Build { self.clear_if_dirty(&out_dir, &self.rustc(compiler)); } diff --git a/standalonex/src/bootstrap/src/core/builder/mod.rs b/standalonex/src/bootstrap/src/core/builder/mod.rs index d59e0fa7..0a1920ad 100644 --- a/standalonex/src/bootstrap/src/core/builder/mod.rs +++ b/standalonex/src/bootstrap/src/core/builder/mod.rs @@ -1,3 +1,4 @@ +use crate::prelude::*; mod cargo; use std::any::{Any, type_name}; @@ -15,10 +16,11 @@ use clap::ValueEnum; pub use self::cargo::Cargo; pub use crate::Compiler; +pub use crate::Subcommand; use crate::core::build_steps::{ check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, tool, vendor, }; -use crate::core::config::flags::Subcommand; + use crate::core::config::{DryRun, TargetSelection}; use crate::utils::cache::Cache; use crate::utils::exec::{BootstrapCommand, command}; @@ -1293,7 +1295,9 @@ impl<'a> Builder<'a> { /// Gets a path to the compiler specified. pub fn rustc(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { + if let Some(path) = &self.build.config.initial_rustc { + path.clone() + } else if compiler.is_snapshot(self) { self.initial_rustc.clone() } else { self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) @@ -1403,7 +1407,7 @@ impl<'a> Builder<'a> { /// Note that this returns `None` if LLVM is disabled, or if we're in a /// check build or dry-run, where there's no need to build all of LLVM. fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() { + if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run { let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); if llvm_config.is_file() { return Some(llvm_config); @@ -1452,7 +1456,7 @@ impl<'a> Builder<'a> { (out, dur - deps) }; - if self.config.print_step_timings && !self.config.dry_run() { + if self.config.print_step_timings && !self.config.dry_run { let step_string = format!("{step:?}"); let brace_index = step_string.find('{').unwrap_or(0); let type_string = type_name::(); @@ -1526,7 +1530,7 @@ impl<'a> Builder<'a> { } pub(crate) fn open_in_browser(&self, path: impl AsRef) { - if self.config.dry_run() || !self.config.cmd.open() { + if self.config.dry_run || !self.config.cmd.open() { return; } diff --git a/standalonex/src/bootstrap/src/core/config/build.rs b/standalonex/src/bootstrap/src/core/config/build.rs new file mode 100644 index 00000000..1a5e4e3f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/build.rs @@ -0,0 +1,59 @@ +use crate::prelude::*; + +use config_macros::define_config; + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option = "build", + src: Option = "src", + host: Option> = "host", + target: Option> = "target", + build_dir: Option = "build-dir", + cargo: Option = "cargo", + rustc: Option = "rustc", + rustfmt: Option = "rustfmt", + cargo_clippy: Option = "cargo-clippy", + docs: Option = "docs", + compiler_docs: Option = "compiler-docs", + library_docs_private_items: Option = "library-docs-private-items", + docs_minification: Option = "docs-minification", + submodules: Option = "submodules", + gdb: Option = "gdb", + lldb: Option = "lldb", + nodejs: Option = "nodejs", + npm: Option = "npm", + python: Option = "python", + reuse: Option = "reuse", + locked_deps: Option = "locked-deps", + vendor: Option = "vendor", + full_bootstrap: Option = "full-bootstrap", + bootstrap_cache_path: Option = "bootstrap-cache-path", + extended: Option = "extended", + tools: Option> = "tools", + verbose: Option = "verbose", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + cargo_native_static: Option = "cargo-native-static", + low_priority: Option = "low-priority", + configure_args: Option> = "configure-args", + local_rebuild: Option = "local-rebuild", + print_step_timings: Option = "print-step-timings", + print_step_rusage: Option = "print-step-rusage", + check_stage: Option = "check-stage", + doc_stage: Option = "doc-stage", + build_stage: Option = "build-stage", + test_stage: Option = "test-stage", + install_stage: Option = "install-stage", + dist_stage: Option = "dist-stage", + bench_stage: Option = "bench-stage", + patch_binaries_for_nix: Option = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally + metrics: Option = "metrics", + android_ndk: Option = "android-ndk", + optimized_compiler_builtins: Option = "optimized-compiler-builtins", + jobs: Option = "jobs", + compiletest_diff_tool: Option = "compiletest-diff-tool", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/changeid.rs b/standalonex/src/bootstrap/src/core/config/changeid.rs new file mode 100644 index 00000000..37b0d273 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/changeid.rs @@ -0,0 +1,11 @@ +use crate::prelude::*; + +/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type +/// for the "change-id" field to parse it even if other fields are invalid. This ensures +/// that if deserialization fails due to other fields, we can still provide the changelogs +/// to allow developers to potentially find the reason for the failure in the logs.. +#[derive(Deserialize, Default)] +pub(crate) struct ChangeIdWrapper { + #[serde(alias = "change-id")] + pub(crate) inner: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config/ci.rs b/standalonex/src/bootstrap/src/core/config/ci.rs new file mode 100644 index 00000000..4046d5ac --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/ci.rs @@ -0,0 +1,13 @@ +use config_macros::define_config; + +define_config! { + /// TOML representation of CI-related paths and settings. + #[derive(Default)] + struct Ci { + channel_file: Option = "channel-file", + version_file: Option = "version-file", + tools_dir: Option = "tools-dir", + llvm_project_dir: Option = "llvm-project-dir", + gcc_dir: Option = "gcc-dir", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/ciconfig.rs b/standalonex/src/bootstrap/src/core/config/ciconfig.rs new file mode 100644 index 00000000..a6da8270 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/ciconfig.rs @@ -0,0 +1,10 @@ +use crate::prelude::*; +/// Configuration for CI-related paths and settings. +#[derive(Debug, Default, Clone)] +pub struct CiConfig { + pub channel_file: PathBuf, + pub version_file: PathBuf, + pub tools_dir: PathBuf, + pub llvm_project_dir: PathBuf, + pub gcc_dir: PathBuf, +} diff --git a/standalonex/src/bootstrap/src/core/config/color.rs b/standalonex/src/bootstrap/src/core/config/color.rs new file mode 100644 index 00000000..74bd4f88 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/color.rs @@ -0,0 +1,8 @@ +use crate::prelude::*; +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Color { + Always, + Never, + #[default] + Auto, +} diff --git a/standalonex/src/bootstrap/src/core/config/config.rs b/standalonex/src/bootstrap/src/core/config/config.rs index d327a465..e0af4300 100644 --- a/standalonex/src/bootstrap/src/core/config/config.rs +++ b/standalonex/src/bootstrap/src/core/config/config.rs @@ -56,3151 +56,107 @@ macro_rules! check_ci_llvm { }; } -/// This file is embedded in the overlay directory of the tarball sources. It is -/// useful in scenarios where developers want to see how the tarball sources were -/// generated. -/// -/// We also use this file to compare the host's config.toml against the CI rustc builder -/// configuration to detect any incompatible options. -pub(crate) const BUILDER_CONFIG_FILENAME: &str = "builder-config"; - -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - -#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] -pub enum DebuginfoLevel { - #[default] - None, - LineDirectivesOnly, - LineTablesOnly, - Limited, - Full, -} - -// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only -// deserializes i64, and derive() only generates visit_u64 -impl<'de> Deserialize<'de> for DebuginfoLevel { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - use serde::de::Error; - - Ok(match Deserialize::deserialize(deserializer)? { - StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, - StringOrInt::Int(0) => DebuginfoLevel::None, - StringOrInt::String(s) if s == "line-directives-only" => { - DebuginfoLevel::LineDirectivesOnly - } - StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, - StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, - StringOrInt::Int(1) => DebuginfoLevel::Limited, - StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, - StringOrInt::Int(2) => DebuginfoLevel::Full, - StringOrInt::Int(n) => { - let other = serde::de::Unexpected::Signed(n); - return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); - } - StringOrInt::String(s) => { - let other = serde::de::Unexpected::Str(&s); - return Err(D::Error::invalid_value( - other, - &"expected none, line-tables-only, limited, or full", - )); - } - }) - } -} - -/// Suitable for passing to `-C debuginfo` -impl Display for DebuginfoLevel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use DebuginfoLevel::*; - f.write_str(match self { - None => "0", - LineDirectivesOnly => "line-directives-only", - LineTablesOnly => "line-tables-only", - Limited => "1", - Full => "2", - }) - } -} - -/// LLD in bootstrap works like this: -/// - Self-contained lld: use `rust-lld` from the compiler's sysroot -/// - External: use an external `lld` binary -/// -/// It is configured depending on the target: -/// 1) Everything except MSVC -/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` -/// - External: `-Clinker-flavor=gnu-lld-cc` -/// 2) MSVC -/// - Self-contained: `-Clinker=` -/// - External: `-Clinker=lld` -#[derive(Copy, Clone, Default, Debug, PartialEq)] -pub enum LldMode { - /// Do not use LLD - #[default] - Unused, - /// Use `rust-lld` from the compiler's sysroot - SelfContained, - /// Use an externally provided `lld` binary. - /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has - /// to be in $PATH. - External, -} - -impl LldMode { - pub fn is_used(&self) -> bool { - match self { - LldMode::SelfContained | LldMode::External => true, - LldMode::Unused => false, - } - } -} - -/// Global configuration for the entire build and/or bootstrap. -/// -/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. -/// -/// Note that this structure is not decoded directly into, but rather it is -/// filled out from the decoded forms of the structs below. For documentation -/// each field, see the corresponding fields in -/// `config.example.toml`. -#[derive(Default, Clone)] -pub struct Config { - pub change_id: Option, - pub bypass_bootstrap_lock: bool, - pub ccache: Option, - /// Call Build::ninja() instead of this. - pub ninja_in_file: bool, - pub verbose: usize, - pub submodules: Option, - pub compiler_docs: bool, - pub library_docs_private_items: bool, - pub docs_minification: bool, - pub docs: bool, - pub locked_deps: bool, - pub vendor: bool, - pub target_config: HashMap, - pub full_bootstrap: bool, - pub bootstrap_cache_path: Option, - pub extended: bool, - pub tools: Option>, - pub sanitizers: bool, - pub profiler: bool, - pub omit_git_hash: bool, - pub skip: Vec, - pub include_default_paths: bool, - pub rustc_error_format: Option, - pub json_output: bool, - pub test_compare_mode: bool, - pub color: Color, - pub patch_binaries_for_nix: Option, - pub stage0_metadata: build_helper::stage0_parser::Stage0, - pub android_ndk: Option, - /// Whether to use the `c` feature of the `compiler_builtins` crate. - pub optimized_compiler_builtins: bool, - - pub stdout_is_tty: bool, - pub stderr_is_tty: bool, - - pub on_fail: Option, - pub stage: u32, - pub keep_stage: Vec, - pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, - pub jobs: Option, - pub cmd: Subcommand, - pub incremental: bool, - pub dry_run: DryRun, - pub dump_bootstrap_shims: bool, - /// Arguments appearing after `--` to be forwarded to tools, - /// e.g. `--fix-broken` or test arguments. - pub free_args: Vec, - - /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. - #[cfg(not(test))] - download_rustc_commit: Option, - #[cfg(test)] - pub download_rustc_commit: Option, - - pub deny_warnings: bool, - pub backtrace_on_ice: bool, - - // llvm codegen options - pub llvm_assertions: bool, - pub llvm_tests: bool, - pub llvm_enzyme: bool, - pub llvm_offload: bool, - pub llvm_plugins: bool, - pub llvm_optimize: bool, - pub llvm_thin_lto: bool, - pub llvm_release_debuginfo: bool, - pub llvm_static_stdcpp: bool, - pub llvm_libzstd: bool, - /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. - #[cfg(not(test))] - llvm_link_shared: Cell>, - #[cfg(test)] - pub llvm_link_shared: Cell>, - pub llvm_clang_cl: Option, - pub llvm_targets: Option, - pub llvm_experimental_targets: Option, - pub llvm_link_jobs: Option, - pub llvm_version_suffix: Option, - pub llvm_use_linker: Option, - pub llvm_allow_old_toolchain: bool, - pub llvm_polly: bool, - pub llvm_clang: bool, - pub llvm_enable_warnings: bool, - pub llvm_from_ci: bool, - pub llvm_build_config: HashMap, - pub llvm_enable_projects: Option, - - pub lld_mode: LldMode, - pub lld_enabled: bool, - pub llvm_tools_enabled: bool, - pub llvm_bitcode_linker_enabled: bool, - - pub llvm_cflags: Option, - pub llvm_cxxflags: Option, - pub llvm_ldflags: Option, - pub llvm_use_libcxx: bool, - - // rust codegen options - pub rust_optimize: RustOptimize, - pub rust_codegen_units: Option, - pub rust_codegen_units_std: Option, - - pub rustc_debug_assertions: bool, - pub std_debug_assertions: bool, - - pub rust_overflow_checks: bool, - pub rust_overflow_checks_std: bool, - pub rust_debug_logging: bool, - pub rust_debuginfo_level_rustc: DebuginfoLevel, - pub rust_debuginfo_level_std: DebuginfoLevel, - pub rust_debuginfo_level_tools: DebuginfoLevel, - pub rust_debuginfo_level_tests: DebuginfoLevel, - pub rust_rpath: bool, - pub rust_strip: bool, - pub rust_frame_pointers: bool, - pub rust_stack_protector: Option, - pub rustc_default_linker: Option, - pub rust_optimize_tests: bool, - pub rust_dist_src: bool, - pub rust_codegen_backends: Vec, - pub rust_verify_llvm_ir: bool, - pub rust_thin_lto_import_instr_limit: Option, - pub rust_randomize_layout: bool, - pub rust_remap_debuginfo: bool, - pub rust_new_symbol_mangling: Option, - pub rust_profile_use: Option, - pub rust_profile_generate: Option, - pub rust_lto: RustcLto, - pub rust_validate_mir_opts: Option, - pub rust_std_features: BTreeSet, - pub llvm_profile_use: Option, - pub llvm_profile_generate: bool, - pub llvm_libunwind_default: Option, - pub enable_bolt_settings: bool, - - pub reproducible_artifacts: Vec, - - pub build: TargetSelection, - pub hosts: Vec, - pub targets: Vec, - pub local_rebuild: bool, - pub jemalloc: bool, - pub control_flow_guard: bool, - pub ehcont_guard: bool, - - // dist misc - pub dist_sign_folder: Option, - pub dist_upload_addr: Option, - pub dist_compression_formats: Option>, - pub dist_compression_profile: String, - pub dist_include_mingw_linker: bool, - pub dist_vendor: bool, - - // libstd features - pub backtrace: bool, // support for RUST_BACKTRACE - - // misc - pub low_priority: bool, - pub channel: String, - pub description: Option, - pub verbose_tests: bool, - pub save_toolstates: Option, - pub print_step_timings: bool, - pub print_step_rusage: bool, - - // Fallback musl-root for all targets - pub musl_root: Option, - pub prefix: Option, - pub sysconfdir: Option, - pub datadir: Option, - pub docdir: Option, - pub bindir: PathBuf, - pub libdir: Option, - pub mandir: Option, - pub codegen_tests: bool, - pub nodejs: Option, - pub npm: Option, - pub gdb: Option, - pub lldb: Option, - pub python: Option, - pub reuse: Option, - pub cargo_native_static: bool, - pub configure_args: Vec, - pub out: PathBuf, - pub rust_info: channel::GitInfo, - - pub cargo_info: channel::GitInfo, - pub rust_analyzer_info: channel::GitInfo, - pub clippy_info: channel::GitInfo, - pub miri_info: channel::GitInfo, - pub rustfmt_info: channel::GitInfo, - pub enzyme_info: channel::GitInfo, - pub in_tree_llvm_info: channel::GitInfo, - pub in_tree_gcc_info: channel::GitInfo, - - // These are either the stage0 downloaded binaries or the locally installed ones. - pub initial_cargo: PathBuf, - pub initial_rustc: PathBuf, - pub initial_cargo_clippy: Option, - - #[cfg(not(test))] - initial_rustfmt: RefCell, - #[cfg(test)] - pub initial_rustfmt: RefCell, - - /// The paths to work with. For example: with `./x check foo bar` we get - /// `paths=["foo", "bar"]`. - pub paths: Vec, - - /// Command for visual diff display, e.g. `diff-tool --color=always`. - pub compiletest_diff_tool: Option, -} - -#[derive(Clone, Debug, Default)] -pub enum RustfmtState { - SystemToolchain(PathBuf), - Downloaded(PathBuf), - Unavailable, - #[default] - LazyEvaluated, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub enum LlvmLibunwind { - #[default] - No, - InTree, - System, -} - -impl FromStr for LlvmLibunwind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "no" => Ok(Self::No), - "in-tree" => Ok(Self::InTree), - "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), - } - } -} - -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SplitDebuginfo { - Packed, - Unpacked, - #[default] - Off, -} - -impl std::str::FromStr for SplitDebuginfo { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "packed" => Ok(SplitDebuginfo::Packed), - "unpacked" => Ok(SplitDebuginfo::Unpacked), - "off" => Ok(SplitDebuginfo::Off), - _ => Err(()), - } - } -} - -impl SplitDebuginfo { - /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `config.example.toml`. - fn default_for_platform(target: TargetSelection) -> Self { - if target.contains("apple") { - SplitDebuginfo::Unpacked - } else if target.is_windows() { - SplitDebuginfo::Packed - } else { - SplitDebuginfo::Off - } - } -} - -/// LTO mode used for compiling rustc itself. -#[derive(Default, Clone, PartialEq, Debug)] -pub enum RustcLto { - Off, - #[default] - ThinLocal, - Thin, - Fat, -} - -impl std::str::FromStr for RustcLto { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "thin-local" => Ok(RustcLto::ThinLocal), - "thin" => Ok(RustcLto::Thin), - "fat" => Ok(RustcLto::Fat), - "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {s}")), - } - } -} - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. -// Making !Copy is highly nontrivial! -pub struct TargetSelection { - pub triple: Interned, - file: Option>, - synthetic: bool, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file, synthetic: false } - } - - pub fn create_synthetic(triple: &str, file: &str) -> Self { - Self { - triple: INTERNER.intern_str(triple), - file: Some(INTERNER.intern_str(file)), - synthetic: true, - } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } - - // See src/bootstrap/synthetic_targets.rs - pub fn is_synthetic(&self) -> bool { - self.synthetic - } - - pub fn is_msvc(&self) -> bool { - self.contains("msvc") - } - - pub fn is_windows(&self) -> bool { - self.contains("windows") - } - - pub fn is_windows_gnu(&self) -> bool { - self.ends_with("windows-gnu") - } - - /// Path to the file defining the custom target, if any. - pub fn filepath(&self) -> Option<&Path> { - self.file.as_ref().map(Path::new) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({file})")?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - -// Targets are often used as directory names throughout bootstrap. -// This impl makes it more ergonomics to use them as such. -impl AsRef for TargetSelection { - fn as_ref(&self) -> &Path { - self.triple.as_ref() - } -} - -/// Per-target configuration stored in the global configuration structure. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct Target { - /// Some(path to llvm-config) if using an external LLVM. - pub llvm_config: Option, - pub llvm_has_rust_patches: Option, - /// Some(path to FileCheck) if one was specified. - pub llvm_filecheck: Option, - pub llvm_libunwind: Option, - pub cc: Option, - pub cxx: Option, - pub ar: Option, - pub ranlib: Option, - pub default_linker: Option, - pub linker: Option, - pub split_debuginfo: Option, - pub sanitizers: Option, - pub profiler: Option, - pub rpath: Option, - pub crt_static: Option, - pub musl_root: Option, - pub musl_libdir: Option, - pub wasi_root: Option, - pub qemu_rootfs: Option, - pub runner: Option, - pub no_std: bool, - pub codegen_backends: Option>, -} - -impl Target { - pub fn from_triple(triple: &str) -> Self { - let mut target: Self = Default::default(); - if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { - target.no_std = true; - } - if triple.contains("emscripten") { - target.runner = Some("node".into()); - } - target - } -} -/// Structure of the `config.toml` file that configuration is read from. -/// -/// This structure uses `Decodable` to automatically decode a TOML configuration -/// file into this format, and then this is traversed and written into the above -/// `Config` structure. -#[derive(Deserialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub(crate) struct TomlConfig { - #[serde(flatten)] - change_id: ChangeIdWrapper, - build: Option, - install: Option, - llvm: Option, - rust: Option, - target: Option>, - dist: Option, - profile: Option, -} - -/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type -/// for the "change-id" field to parse it even if other fields are invalid. This ensures -/// that if deserialization fails due to other fields, we can still provide the changelogs -/// to allow developers to potentially find the reason for the failure in the logs.. -#[derive(Deserialize, Default)] -pub(crate) struct ChangeIdWrapper { - #[serde(alias = "change-id")] - pub(crate) inner: Option, -} - -/// Describes how to handle conflicts in merging two [`TomlConfig`] -#[derive(Copy, Clone, Debug)] -enum ReplaceOpt { - /// Silently ignore a duplicated value - IgnoreDuplicate, - /// Override the current value, even if it's `Some` - Override, - /// Exit with an error on duplicate values - ErrorOnDuplicate, -} - -trait Merge { - fn merge(&mut self, other: Self, replace: ReplaceOpt); -} - -impl Merge for TomlConfig { - fn merge( - &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self, - replace: ReplaceOpt, - ) { - fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { - if let Some(new) = y { - if let Some(original) = x { - original.merge(new, replace); - } else { - *x = Some(new); - } - } - } - self.change_id.inner.merge(change_id.inner, replace); - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - - match (self.target.as_mut(), target) { - (_, None) => {} - (None, Some(target)) => self.target = Some(target), - (Some(original_target), Some(new_target)) => { - for (triple, new) in new_target { - if let Some(original) = original_target.get_mut(&triple) { - original.merge(new, replace); - } else { - original_target.insert(triple, new); - } - } - } - } - } -} - -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap. -macro_rules! define_config { - ($(#[$attr:meta])* struct $name:ident { - $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* - }) => { - $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* - } - - impl Merge for $name { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - $( - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.$field.is_none() { - self.$field = other.$field; - } - }, - ReplaceOpt::Override => { - if other.$field.is_some() { - self.$field = other.$field; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.$field.is_some() { - if self.$field.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option: `{}`", stringify!($field)); - exit!(2); - } - } else { - self.$field = other.$field; - } - } - } - } - )* - } - } - - // The following is a trimmed version of what serde_derive generates. All parts not relevant - // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of bootstrap. - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct Field; - impl<'de> serde::de::Visitor<'de> for Field { - type Value = $name; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(concat!("struct ", stringify!($name))) - } - - #[inline] - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - $(let mut $field: Option<$field_ty> = None;)* - while let Some(key) = - match serde::de::MapAccess::next_key::(&mut map) { - Ok(val) => val, - Err(err) => { - return Err(err); - } - } - { - match &*key { - $($field_key => { - if $field.is_some() { - return Err(::duplicate_field( - $field_key, - )); - } - $field = match serde::de::MapAccess::next_value::<$field_ty>( - &mut map, - ) { - Ok(val) => Some(val), - Err(err) => { - return Err(err); - } - }; - })* - key => { - return Err(serde::de::Error::unknown_field(key, FIELDS)); - } - } - } - Ok($name { $($field),* }) - } - } - const FIELDS: &'static [&'static str] = &[ - $($field_key,)* - ]; - Deserializer::deserialize_struct( - deserializer, - stringify!($name), - FIELDS, - Field, - ) - } - } - } -} - -impl Merge for Option { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.is_none() { - *self = other; - } - } - ReplaceOpt::Override => { - if other.is_some() { - *self = other; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.is_some() { - if self.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option"); - exit!(2); - } - } else { - *self = other; - } - } - } - } - } -} - -define_config! { - /// TOML representation of various global build decisions. - #[derive(Default)] - struct Build { - build: Option = "build", - host: Option> = "host", - target: Option> = "target", - build_dir: Option = "build-dir", - cargo: Option = "cargo", - rustc: Option = "rustc", - rustfmt: Option = "rustfmt", - cargo_clippy: Option = "cargo-clippy", - docs: Option = "docs", - compiler_docs: Option = "compiler-docs", - library_docs_private_items: Option = "library-docs-private-items", - docs_minification: Option = "docs-minification", - submodules: Option = "submodules", - gdb: Option = "gdb", - lldb: Option = "lldb", - nodejs: Option = "nodejs", - npm: Option = "npm", - python: Option = "python", - reuse: Option = "reuse", - locked_deps: Option = "locked-deps", - vendor: Option = "vendor", - full_bootstrap: Option = "full-bootstrap", - bootstrap_cache_path: Option = "bootstrap-cache-path", - extended: Option = "extended", - tools: Option> = "tools", - verbose: Option = "verbose", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - cargo_native_static: Option = "cargo-native-static", - low_priority: Option = "low-priority", - configure_args: Option> = "configure-args", - local_rebuild: Option = "local-rebuild", - print_step_timings: Option = "print-step-timings", - print_step_rusage: Option = "print-step-rusage", - check_stage: Option = "check-stage", - doc_stage: Option = "doc-stage", - build_stage: Option = "build-stage", - test_stage: Option = "test-stage", - install_stage: Option = "install-stage", - dist_stage: Option = "dist-stage", - bench_stage: Option = "bench-stage", - patch_binaries_for_nix: Option = "patch-binaries-for-nix", - // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally - metrics: Option = "metrics", - android_ndk: Option = "android-ndk", - optimized_compiler_builtins: Option = "optimized-compiler-builtins", - jobs: Option = "jobs", - compiletest_diff_tool: Option = "compiletest-diff-tool", - } -} - -define_config! { - /// TOML representation of various global install decisions. - struct Install { - prefix: Option = "prefix", - sysconfdir: Option = "sysconfdir", - docdir: Option = "docdir", - bindir: Option = "bindir", - libdir: Option = "libdir", - mandir: Option = "mandir", - datadir: Option = "datadir", - } -} - -define_config! { - /// TOML representation of how the LLVM build is configured. - struct Llvm { - optimize: Option = "optimize", - thin_lto: Option = "thin-lto", - release_debuginfo: Option = "release-debuginfo", - assertions: Option = "assertions", - tests: Option = "tests", - enzyme: Option = "enzyme", - plugins: Option = "plugins", - ccache: Option = "ccache", - static_libstdcpp: Option = "static-libstdcpp", - libzstd: Option = "libzstd", - ninja: Option = "ninja", - targets: Option = "targets", - experimental_targets: Option = "experimental-targets", - link_jobs: Option = "link-jobs", - link_shared: Option = "link-shared", - version_suffix: Option = "version-suffix", - clang_cl: Option = "clang-cl", - cflags: Option = "cflags", - cxxflags: Option = "cxxflags", - ldflags: Option = "ldflags", - use_libcxx: Option = "use-libcxx", - use_linker: Option = "use-linker", - allow_old_toolchain: Option = "allow-old-toolchain", - offload: Option = "offload", - polly: Option = "polly", - clang: Option = "clang", - enable_warnings: Option = "enable-warnings", - download_ci_llvm: Option = "download-ci-llvm", - build_config: Option> = "build-config", - enable_projects: Option = "enable-projects", - } -} - -define_config! { - struct Dist { - sign_folder: Option = "sign-folder", - upload_addr: Option = "upload-addr", - src_tarball: Option = "src-tarball", - compression_formats: Option> = "compression-formats", - compression_profile: Option = "compression-profile", - include_mingw_linker: Option = "include-mingw-linker", - vendor: Option = "vendor", - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum StringOrBool { - String(String), - Bool(bool), -} - -impl Default for StringOrBool { - fn default() -> StringOrBool { - StringOrBool::Bool(false) - } -} - -impl StringOrBool { - fn is_string_or_true(&self) -> bool { - matches!(self, Self::String(_) | Self::Bool(true)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RustOptimize { - String(String), - Int(u8), - Bool(bool), -} - -impl Default for RustOptimize { - fn default() -> RustOptimize { - RustOptimize::Bool(false) - } -} - -impl<'de> Deserialize<'de> for RustOptimize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(OptimizeVisitor) - } -} - -struct OptimizeVisitor; - -impl serde::de::Visitor<'_> for OptimizeVisitor { - type Value = RustOptimize; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - if matches!(value, "s" | "z") { - Ok(RustOptimize::String(value.to_string())) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - if matches!(value, 0..=3) { - Ok(RustOptimize::Int(value as u8)) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - Ok(RustOptimize::Bool(value)) - } -} - -fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { - format!( - r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# - ) -} - -impl RustOptimize { - pub(crate) fn is_release(&self) -> bool { - match &self { - RustOptimize::Bool(true) | RustOptimize::String(_) => true, - RustOptimize::Int(i) => *i > 0, - RustOptimize::Bool(false) => false, - } - } - - pub(crate) fn get_opt_level(&self) -> Option { - match &self { - RustOptimize::String(s) => Some(s.clone()), - RustOptimize::Int(i) => Some(i.to_string()), - RustOptimize::Bool(_) => None, - } - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum StringOrInt { - String(String), - Int(i64), -} - -impl<'de> Deserialize<'de> for LldMode { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct LldModeVisitor; - - impl serde::de::Visitor<'_> for LldModeVisitor { - type Value = LldMode; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("one of true, 'self-contained' or 'external'") - } - - fn visit_bool(self, v: bool) -> Result - where - E: serde::de::Error, - { - Ok(if v { LldMode::External } else { LldMode::Unused }) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - match v { - "external" => Ok(LldMode::External), - "self-contained" => Ok(LldMode::SelfContained), - _ => Err(E::custom("unknown mode {v}")), - } - } - } - - deserializer.deserialize_any(LldModeVisitor) - } -} - -define_config! { - /// TOML representation of how the Rust build is configured. - struct Rust { - optimize: Option = "optimize", - debug: Option = "debug", - codegen_units: Option = "codegen-units", - codegen_units_std: Option = "codegen-units-std", - rustc_debug_assertions: Option = "debug-assertions", - randomize_layout: Option = "randomize-layout", - std_debug_assertions: Option = "debug-assertions-std", - overflow_checks: Option = "overflow-checks", - overflow_checks_std: Option = "overflow-checks-std", - debug_logging: Option = "debug-logging", - debuginfo_level: Option = "debuginfo-level", - debuginfo_level_rustc: Option = "debuginfo-level-rustc", - debuginfo_level_std: Option = "debuginfo-level-std", - debuginfo_level_tools: Option = "debuginfo-level-tools", - debuginfo_level_tests: Option = "debuginfo-level-tests", - backtrace: Option = "backtrace", - incremental: Option = "incremental", - parallel_compiler: Option = "parallel-compiler", - default_linker: Option = "default-linker", - channel: Option = "channel", - description: Option = "description", - musl_root: Option = "musl-root", - rpath: Option = "rpath", - strip: Option = "strip", - frame_pointers: Option = "frame-pointers", - stack_protector: Option = "stack-protector", - verbose_tests: Option = "verbose-tests", - optimize_tests: Option = "optimize-tests", - codegen_tests: Option = "codegen-tests", - omit_git_hash: Option = "omit-git-hash", - dist_src: Option = "dist-src", - save_toolstates: Option = "save-toolstates", - codegen_backends: Option> = "codegen-backends", - llvm_bitcode_linker: Option = "llvm-bitcode-linker", - lld: Option = "lld", - lld_mode: Option = "use-lld", - llvm_tools: Option = "llvm-tools", - deny_warnings: Option = "deny-warnings", - backtrace_on_ice: Option = "backtrace-on-ice", - verify_llvm_ir: Option = "verify-llvm-ir", - thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", - remap_debuginfo: Option = "remap-debuginfo", - jemalloc: Option = "jemalloc", - test_compare_mode: Option = "test-compare-mode", - llvm_libunwind: Option = "llvm-libunwind", - control_flow_guard: Option = "control-flow-guard", - ehcont_guard: Option = "ehcont-guard", - new_symbol_mangling: Option = "new-symbol-mangling", - profile_generate: Option = "profile-generate", - profile_use: Option = "profile-use", - // ignored; this is set from an env var set by bootstrap.py - download_rustc: Option = "download-rustc", - lto: Option = "lto", - validate_mir_opts: Option = "validate-mir-opts", - std_features: Option> = "std-features", - } -} - -define_config! { - /// TOML representation of how each build target is configured. - struct TomlTarget { - cc: Option = "cc", - cxx: Option = "cxx", - ar: Option = "ar", - ranlib: Option = "ranlib", - default_linker: Option = "default-linker", - linker: Option = "linker", - split_debuginfo: Option = "split-debuginfo", - llvm_config: Option = "llvm-config", - llvm_has_rust_patches: Option = "llvm-has-rust-patches", - llvm_filecheck: Option = "llvm-filecheck", - llvm_libunwind: Option = "llvm-libunwind", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - rpath: Option = "rpath", - crt_static: Option = "crt-static", - musl_root: Option = "musl-root", - musl_libdir: Option = "musl-libdir", - wasi_root: Option = "wasi-root", - qemu_rootfs: Option = "qemu-rootfs", - no_std: Option = "no-std", - codegen_backends: Option> = "codegen-backends", - runner: Option = "runner", - } -} - -impl Config { - pub fn default_opts() -> Config { - Config { - bypass_bootstrap_lock: false, - llvm_optimize: true, - ninja_in_file: true, - llvm_static_stdcpp: false, - llvm_libzstd: false, - backtrace: true, - rust_optimize: RustOptimize::Bool(true), - rust_optimize_tests: true, - rust_randomize_layout: false, - submodules: None, - docs: true, - docs_minification: true, - rust_rpath: true, - rust_strip: false, - channel: "dev".to_string(), - codegen_tests: true, - rust_dist_src: true, - rust_codegen_backends: vec!["llvm".to_owned()], - deny_warnings: true, - bindir: "bin".into(), - dist_include_mingw_linker: true, - dist_compression_profile: "fast".into(), - - stdout_is_tty: std::io::stdout().is_terminal(), - stderr_is_tty: std::io::stderr().is_terminal(), - - // set by build.rs - build: TargetSelection::from_user(env!("BUILD_TRIPLE")), - - src: { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - manifest_dir.parent().unwrap().parent().unwrap().to_owned() - }, - out: PathBuf::from("build"), - - // This is needed by codegen_ssa on macOS to ship `llvm-objcopy` aliased to - // `rust-objcopy` to workaround bad `strip`s on macOS. - llvm_tools_enabled: true, - - ..Default::default() - } - } - - pub(crate) fn get_builder_toml(&self, build_name: &str) -> Result { - if self.dry_run() { - return Ok(TomlConfig::default()); - } - - let builder_config_path = - self.out.join(self.build.triple).join(build_name).join(BUILDER_CONFIG_FILENAME); - Self::get_toml(&builder_config_path) - } - - #[cfg(test)] - pub(crate) fn get_toml(_: &Path) -> Result { - Ok(TomlConfig::default()) - } - - #[cfg(not(test))] - pub(crate) fn get_toml(file: &Path) -> Result { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .inspect_err(|_| { - if let Ok(Some(changes)) = toml::from_str(&contents) - .and_then(|table: toml::Value| ChangeIdWrapper::deserialize(table)) - .map(|change_id| change_id.inner.map(crate::find_recent_config_change_ids)) - { - if !changes.is_empty() { - println!( - "WARNING: There have been changes to x.py since you last updated:\n{}", - crate::human_readable_changes(&changes) - ); - } - } - }) - } - - pub fn parse(flags: Flags) -> Config { - Self::parse_inner(flags, Self::get_toml) - } - - pub(crate) fn parse_inner( - mut flags: Flags, - get_toml: impl Fn(&Path) -> Result, - ) -> Config { - let mut config = Config::default_opts(); - - // Set flags. - config.paths = std::mem::take(&mut flags.paths); - config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); - config.include_default_paths = flags.include_default_paths; - config.rustc_error_format = flags.rustc_error_format; - config.json_output = flags.json_output; - config.on_fail = flags.on_fail; - config.cmd = flags.cmd; - config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; - config.dump_bootstrap_shims = flags.dump_bootstrap_shims; - config.keep_stage = flags.keep_stage; - config.keep_stage_std = flags.keep_stage_std; - config.color = flags.color; - config.free_args = std::mem::take(&mut flags.free_args); - config.llvm_profile_use = flags.llvm_profile_use; - config.llvm_profile_generate = flags.llvm_profile_generate; - config.enable_bolt_settings = flags.enable_bolt_settings; - config.bypass_bootstrap_lock = flags.bypass_bootstrap_lock; - - // Infer the rest of the configuration. - - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely different machine from where it was compiled. - let mut cmd = helpers::git(None); - // NOTE: we cannot support running from outside the repository because the only other path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded rather than compiled locally. - // We still support running outside the repository if we find we aren't in a git directory. - - // NOTE: We get a relative path from git to work around an issue on MSYS/mingw. If we used an absolute path, - // and end up using MSYS's git rather than git-for-windows, we would get a unix-y MSYS path. But as bootstrap - // has already been (kinda-cross-)compiled to Windows land, we require a normal Windows path. - cmd.arg("rev-parse").arg("--show-cdup"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .as_command_mut() - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root_relative = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes, - // and to resolve any relative components. - let git_root = env::current_dir() - .unwrap() - .join(PathBuf::from(git_root_relative.trim())) - .canonicalize() - .unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let git_root = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => git_root, - }; - // If this doesn't have at least `stage0`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if git_root.join("src").join("stage0").exists() { - config.src = git_root; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file(); - - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - let file_content = t!(fs::read_to_string(config.src.join("src/ci/channel"))); - let ci_channel = file_content.trim_end(); - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(if cfg!(not(feature = "bootstrap-self-test")) { - toml_path.canonicalize().unwrap() - } else { - toml_path.clone() - }); - get_toml(&toml_path).unwrap_or_else(|e| { - eprintln!("ERROR: Failed to parse '{}': {e}", toml_path.display()); - exit!(2); - }) - } else { - config.config = None; - TomlConfig::default() - }; - - if cfg!(test) { - // When configuring bootstrap for tests, make sure to set the rustc and Cargo to the - // same ones used to call the tests (if custom ones are not defined in the toml). If we - // don't do that, bootstrap will use its own detection logic to find a suitable rustc - // and Cargo, which doesn't work when the caller is specÃŦfying a custom local rustc or - // Cargo in their config.toml. - let build = toml.build.get_or_insert_with(Default::default); - build.rustc = build.rustc.take().or(std::env::var_os("RUSTC").map(|p| p.into())); - build.cargo = build.cargo.take().or(std::env::var_os("CARGO").map(|p| p.into())); - } - - if let Some(include) = &toml.profile { - // Allows creating alias for profile names, allowing - // profiles to be renamed while maintaining back compatibility - // Keep in sync with `profile_aliases` in bootstrap.py - let profile_aliases = HashMap::from([("user", "dist")]); - let include = match profile_aliases.get(include.as_str()) { - Some(alias) => alias, - None => include.as_str(), - }; - let mut include_path = config.src.clone(); - include_path.push("src"); - include_path.push("bootstrap"); - include_path.push("defaults"); - include_path.push(format!("config.{include}.toml")); - let included_toml = get_toml(&include_path).unwrap_or_else(|e| { - eprintln!( - "ERROR: Failed to parse default config profile at '{}': {e}", - include_path.display() - ); - exit!(2); - }); - toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); - } - let mut override_toml = TomlConfig::default(); - for option in flags.set.iter() { - fn get_table(option: &str) -> Result { - toml::from_str(option).and_then(|table: toml::Value| TomlConfig::deserialize(table)) - } - - let mut err = match get_table(option) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => e, - }; - // We want to be able to set string values without quotes, - // like in `configure.py`. Try adding quotes around the right hand side - if let Some((key, value)) = option.split_once('=') { - if !value.contains('"') { - match get_table(&format!(r#"{key}="{value}""#)) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => err = e, - } - } - } - eprintln!("failed to parse override `{option}`: `{err}"); - exit!(2) - } - toml.merge(override_toml, ReplaceOpt::Override); - - config.change_id = toml.change_id.inner; - - let Build { - build, - host, - target, - build_dir, - cargo, - rustc, - rustfmt, - cargo_clippy, - docs, - compiler_docs, - library_docs_private_items, - docs_minification, - submodules, - gdb, - lldb, - nodejs, - npm, - python, - reuse, - locked_deps, - vendor, - full_bootstrap, - bootstrap_cache_path, - extended, - tools, - verbose, - sanitizers, - profiler, - cargo_native_static, - low_priority, - configure_args, - local_rebuild, - print_step_timings, - print_step_rusage, - check_stage, - doc_stage, - build_stage, - test_stage, - install_stage, - dist_stage, - bench_stage, - patch_binaries_for_nix, - // This field is only used by bootstrap.py - metrics: _, - android_ndk, - optimized_compiler_builtins, - jobs, - compiletest_diff_tool, - } = toml.build.unwrap_or_default(); - - config.jobs = Some(threads_from_config(flags.jobs.unwrap_or(jobs.unwrap_or(0)))); - - if let Some(file_build) = build { - config.build = TargetSelection::from_user(&file_build); - }; - - set(&mut config.out, flags.build_dir.or_else(|| build_dir.map(PathBuf::from))); - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = absolute(&config.out).expect("can't make empty path absolute"); - } - - if cargo_clippy.is_some() && rustc.is_none() { - println!( - "WARNING: Using `build.cargo-clippy` without `build.rustc` usually fails due to toolchain conflict." - ); - } - - config.initial_cargo_clippy = cargo_clippy; - - config.initial_rustc = if let Some(rustc) = rustc { - if !flags.skip_stage0_validation { - config.check_stage0_version(&rustc, "rustc"); - } - rustc - } else { - config.download_beta_toolchain(); - config - .out - .join(config.build) - .join("stage0") - .join("bin") - .join(exe("rustc", config.build)) - }; - - config.initial_cargo = if let Some(cargo) = cargo { - if !flags.skip_stage0_validation { - config.check_stage0_version(&cargo, "cargo"); - } - cargo - } else { - config.download_beta_toolchain(); - config - .out - .join(config.build) - .join("stage0") - .join("bin") - .join(exe("cargo", config.build)) - }; - - // NOTE: it's important this comes *after* we set `initial_rustc` just above. - if config.dry_run() { - let dir = config.out.join("tmp-dry-run"); - t!(fs::create_dir_all(&dir)); - config.out = dir; - } - - config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { - arg_host - } else if let Some(file_host) = host { - file_host.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - vec![config.build] - }; - config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { - arg_target - } else if let Some(file_target) = target { - file_target.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - // If target is *not* configured, then default to the host - // toolchains. - config.hosts.clone() - }; - - config.nodejs = nodejs.map(PathBuf::from); - config.npm = npm.map(PathBuf::from); - config.gdb = gdb.map(PathBuf::from); - config.lldb = lldb.map(PathBuf::from); - config.python = python.map(PathBuf::from); - config.reuse = reuse.map(PathBuf::from); - config.submodules = submodules; - config.android_ndk = android_ndk; - config.bootstrap_cache_path = bootstrap_cache_path; - set(&mut config.low_priority, low_priority); - set(&mut config.compiler_docs, compiler_docs); - set(&mut config.library_docs_private_items, library_docs_private_items); - set(&mut config.docs_minification, docs_minification); - set(&mut config.docs, docs); - set(&mut config.locked_deps, locked_deps); - set(&mut config.vendor, vendor); - set(&mut config.full_bootstrap, full_bootstrap); - set(&mut config.extended, extended); - config.tools = tools; - set(&mut config.verbose, verbose); - set(&mut config.sanitizers, sanitizers); - set(&mut config.profiler, profiler); - set(&mut config.cargo_native_static, cargo_native_static); - set(&mut config.configure_args, configure_args); - set(&mut config.local_rebuild, local_rebuild); - set(&mut config.print_step_timings, print_step_timings); - set(&mut config.print_step_rusage, print_step_rusage); - config.patch_binaries_for_nix = patch_binaries_for_nix; - - config.verbose = cmp::max(config.verbose, flags.verbose as usize); - - // Verbose flag is a good default for `rust.verbose-tests`. - config.verbose_tests = config.is_verbose(); - - if let Some(install) = toml.install { - let Install { prefix, sysconfdir, docdir, bindir, libdir, mandir, datadir } = install; - config.prefix = prefix.map(PathBuf::from); - config.sysconfdir = sysconfdir.map(PathBuf::from); - config.datadir = datadir.map(PathBuf::from); - config.docdir = docdir.map(PathBuf::from); - set(&mut config.bindir, bindir.map(PathBuf::from)); - config.libdir = libdir.map(PathBuf::from); - config.mandir = mandir.map(PathBuf::from); - } - - config.llvm_assertions = - toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false)); - - // Store off these values as options because if they're not provided - // we'll infer default values for them later - let mut llvm_tests = None; - let mut llvm_enzyme = None; - let mut llvm_offload = None; - let mut llvm_plugins = None; - let mut debug = None; - let mut rustc_debug_assertions = None; - let mut std_debug_assertions = None; - let mut overflow_checks = None; - let mut overflow_checks_std = None; - let mut debug_logging = None; - let mut debuginfo_level = None; - let mut debuginfo_level_rustc = None; - let mut debuginfo_level_std = None; - let mut debuginfo_level_tools = None; - let mut debuginfo_level_tests = None; - let mut optimize = None; - let mut lld_enabled = None; - let mut std_features = None; - - let is_user_configured_rust_channel = - if let Some(channel) = toml.rust.as_ref().and_then(|r| r.channel.clone()) { - config.channel = channel; - true - } else { - false - }; - - let default = config.channel == "dev"; - config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(default); - - config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - config.cargo_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/cargo")); - config.rust_analyzer_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer")); - config.clippy_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/clippy")); - config.miri_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/miri")); - config.rustfmt_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rustfmt")); - config.enzyme_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/enzyme")); - config.in_tree_llvm_info = GitInfo::new(false, &config.src.join("src/llvm-project")); - config.in_tree_gcc_info = GitInfo::new(false, &config.src.join("src/gcc")); - - if let Some(rust) = toml.rust { - let Rust { - optimize: optimize_toml, - debug: debug_toml, - codegen_units, - codegen_units_std, - rustc_debug_assertions: rustc_debug_assertions_toml, - std_debug_assertions: std_debug_assertions_toml, - overflow_checks: overflow_checks_toml, - overflow_checks_std: overflow_checks_std_toml, - debug_logging: debug_logging_toml, - debuginfo_level: debuginfo_level_toml, - debuginfo_level_rustc: debuginfo_level_rustc_toml, - debuginfo_level_std: debuginfo_level_std_toml, - debuginfo_level_tools: debuginfo_level_tools_toml, - debuginfo_level_tests: debuginfo_level_tests_toml, - backtrace, - incremental, - parallel_compiler, - randomize_layout, - default_linker, - channel: _, // already handled above - description, - musl_root, - rpath, - verbose_tests, - optimize_tests, - codegen_tests, - omit_git_hash: _, // already handled above - dist_src, - save_toolstates, - codegen_backends, - lld: lld_enabled_toml, - llvm_tools, - llvm_bitcode_linker, - deny_warnings, - backtrace_on_ice, - verify_llvm_ir, - thin_lto_import_instr_limit, - remap_debuginfo, - jemalloc, - test_compare_mode, - llvm_libunwind, - control_flow_guard, - ehcont_guard, - new_symbol_mangling, - profile_generate, - profile_use, - download_rustc, - lto, - validate_mir_opts, - frame_pointers, - stack_protector, - strip, - lld_mode, - std_features: std_features_toml, - } = rust; - - config.download_rustc_commit = - config.download_ci_rustc_commit(download_rustc, config.llvm_assertions); - - debug = debug_toml; - rustc_debug_assertions = rustc_debug_assertions_toml; - std_debug_assertions = std_debug_assertions_toml; - overflow_checks = overflow_checks_toml; - overflow_checks_std = overflow_checks_std_toml; - debug_logging = debug_logging_toml; - debuginfo_level = debuginfo_level_toml; - debuginfo_level_rustc = debuginfo_level_rustc_toml; - debuginfo_level_std = debuginfo_level_std_toml; - debuginfo_level_tools = debuginfo_level_tools_toml; - debuginfo_level_tests = debuginfo_level_tests_toml; - lld_enabled = lld_enabled_toml; - std_features = std_features_toml; - - optimize = optimize_toml; - config.rust_new_symbol_mangling = new_symbol_mangling; - set(&mut config.rust_optimize_tests, optimize_tests); - set(&mut config.codegen_tests, codegen_tests); - set(&mut config.rust_rpath, rpath); - set(&mut config.rust_strip, strip); - set(&mut config.rust_frame_pointers, frame_pointers); - config.rust_stack_protector = stack_protector; - set(&mut config.jemalloc, jemalloc); - set(&mut config.test_compare_mode, test_compare_mode); - set(&mut config.backtrace, backtrace); - config.description = description; - set(&mut config.rust_dist_src, dist_src); - set(&mut config.verbose_tests, verbose_tests); - // in the case "false" is set explicitly, do not overwrite the command line args - if let Some(true) = incremental { - config.incremental = true; - } - set(&mut config.lld_mode, lld_mode); - set(&mut config.llvm_bitcode_linker_enabled, llvm_bitcode_linker); - - config.rust_randomize_layout = randomize_layout.unwrap_or_default(); - config.llvm_tools_enabled = llvm_tools.unwrap_or(true); - - // FIXME: Remove this option at the end of 2024. - if parallel_compiler.is_some() { - println!( - "WARNING: The `rust.parallel-compiler` option is deprecated and does nothing. The parallel compiler (with one thread) is now the default" - ); - } - - config.llvm_enzyme = - llvm_enzyme.unwrap_or(config.channel == "dev" || config.channel == "nightly"); - config.rustc_default_linker = default_linker; - config.musl_root = musl_root.map(PathBuf::from); - config.save_toolstates = save_toolstates.map(PathBuf::from); - set(&mut config.deny_warnings, match flags.warnings { - Warnings::Deny => Some(true), - Warnings::Warn => Some(false), - Warnings::Default => deny_warnings, - }); - set(&mut config.backtrace_on_ice, backtrace_on_ice); - set(&mut config.rust_verify_llvm_ir, verify_llvm_ir); - config.rust_thin_lto_import_instr_limit = thin_lto_import_instr_limit; - set(&mut config.rust_remap_debuginfo, remap_debuginfo); - set(&mut config.control_flow_guard, control_flow_guard); - set(&mut config.ehcont_guard, ehcont_guard); - config.llvm_libunwind_default = - llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - config.rust_codegen_backends = backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect(); - } - - config.rust_codegen_units = codegen_units.map(threads_from_config); - config.rust_codegen_units_std = codegen_units_std.map(threads_from_config); - config.rust_profile_use = flags.rust_profile_use.or(profile_use); - config.rust_profile_generate = flags.rust_profile_generate.or(profile_generate); - config.rust_lto = - lto.as_deref().map(|value| RustcLto::from_str(value).unwrap()).unwrap_or_default(); - config.rust_validate_mir_opts = validate_mir_opts; - } else { - config.rust_profile_use = flags.rust_profile_use; - config.rust_profile_generate = flags.rust_profile_generate; - } - - config.reproducible_artifacts = flags.reproducible_artifact; - - // We need to override `rust.channel` if it's manually specified when using the CI rustc. - // This is because if the compiler uses a different channel than the one specified in config.toml, - // tests may fail due to using a different channel than the one used by the compiler during tests. - if let Some(commit) = &config.download_rustc_commit { - if is_user_configured_rust_channel { - println!( - "WARNING: `rust.download-rustc` is enabled. The `rust.channel` option will be overridden by the CI rustc's channel." - ); - - let channel = config - .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) - .trim() - .to_owned(); - - config.channel = channel; - } - } else if config.rust_info.is_from_tarball() && !is_user_configured_rust_channel { - ci_channel.clone_into(&mut config.channel); - } - - if let Some(llvm) = toml.llvm { - let Llvm { - optimize: optimize_toml, - thin_lto, - release_debuginfo, - assertions: _, - tests, - enzyme, - plugins, - ccache, - static_libstdcpp, - libzstd, - ninja, - targets, - experimental_targets, - link_jobs, - link_shared, - version_suffix, - clang_cl, - cflags, - cxxflags, - ldflags, - use_libcxx, - use_linker, - allow_old_toolchain, - offload, - polly, - clang, - enable_warnings, - download_ci_llvm, - build_config, - enable_projects, - } = llvm; - match ccache { - Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), - Some(StringOrBool::Bool(true)) => { - config.ccache = Some("ccache".to_string()); - } - Some(StringOrBool::Bool(false)) | None => {} - } - set(&mut config.ninja_in_file, ninja); - llvm_tests = tests; - llvm_enzyme = enzyme; - llvm_offload = offload; - llvm_plugins = plugins; - set(&mut config.llvm_optimize, optimize_toml); - set(&mut config.llvm_thin_lto, thin_lto); - set(&mut config.llvm_release_debuginfo, release_debuginfo); - set(&mut config.llvm_static_stdcpp, static_libstdcpp); - set(&mut config.llvm_libzstd, libzstd); - if let Some(v) = link_shared { - config.llvm_link_shared.set(Some(v)); - } - - config.llvm_targets.clone_from(&targets); - config.llvm_experimental_targets.clone_from(&experimental_targets); - config.llvm_link_jobs = link_jobs; - config.llvm_version_suffix.clone_from(&version_suffix); - config.llvm_clang_cl.clone_from(&clang_cl); - config.llvm_enable_projects.clone_from(&enable_projects); - - config.llvm_cflags.clone_from(&cflags); - config.llvm_cxxflags.clone_from(&cxxflags); - config.llvm_ldflags.clone_from(&ldflags); - set(&mut config.llvm_use_libcxx, use_libcxx); - config.llvm_use_linker.clone_from(&use_linker); - config.llvm_allow_old_toolchain = allow_old_toolchain.unwrap_or(false); - config.llvm_offload = offload.unwrap_or(false); - config.llvm_polly = polly.unwrap_or(false); - config.llvm_clang = clang.unwrap_or(false); - config.llvm_enable_warnings = enable_warnings.unwrap_or(false); - config.llvm_build_config = build_config.clone().unwrap_or(Default::default()); - - config.llvm_from_ci = - config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions); - - if config.llvm_from_ci { - let warn = |option: &str| { - println!( - "WARNING: `{option}` will only be used on `compiler/rustc_llvm` build, not for the LLVM build." - ); - println!( - "HELP: To use `{option}` for LLVM builds, set `download-ci-llvm` option to false." - ); - }; - - if static_libstdcpp.is_some() { - warn("static-libstdcpp"); - } - - if link_shared.is_some() { - warn("link-shared"); - } - - // FIXME(#129153): instead of all the ad-hoc `download-ci-llvm` checks that follow, - // use the `builder-config` present in tarballs since #128822 to compare the local - // config to the ones used to build the LLVM artifacts on CI, and only notify users - // if they've chosen a different value. - - if libzstd.is_some() { - println!( - "WARNING: when using `download-ci-llvm`, the local `llvm.libzstd` option, \ - like almost all `llvm.*` options, will be ignored and set by the LLVM CI \ - artifacts builder config." - ); - println!( - "HELP: To use `llvm.libzstd` for LLVM/LLD builds, set `download-ci-llvm` option to false." - ); - } - } - - if !config.llvm_from_ci && config.llvm_thin_lto && link_shared.is_none() { - // If we're building with ThinLTO on, by default we want to link - // to LLVM shared, to avoid re-doing ThinLTO (which happens in - // the link step) with each stage. - config.llvm_link_shared.set(Some(true)); - } - } else { - config.llvm_from_ci = config.parse_download_ci_llvm(None, false); - } - - if let Some(t) = toml.target { - for (triple, cfg) in t { - let mut target = Target::from_triple(&triple); - - if let Some(ref s) = cfg.llvm_config { - if config.download_rustc_commit.is_some() && triple == *config.build.triple { - panic!( - "setting llvm_config for the host is incompatible with download-rustc" - ); - } - target.llvm_config = Some(config.src.join(s)); - } - if let Some(patches) = cfg.llvm_has_rust_patches { - assert!( - config.submodules == Some(false) || cfg.llvm_config.is_some(), - "use of `llvm-has-rust-patches` is restricted to cases where either submodules are disabled or llvm-config been provided" - ); - target.llvm_has_rust_patches = Some(patches); - } - if let Some(ref s) = cfg.llvm_filecheck { - target.llvm_filecheck = Some(config.src.join(s)); - } - target.llvm_libunwind = cfg.llvm_libunwind.as_ref().map(|v| { - v.parse().unwrap_or_else(|_| { - panic!("failed to parse target.{triple}.llvm-libunwind") - }) - }); - if let Some(s) = cfg.no_std { - target.no_std = s; - } - target.cc = cfg.cc.map(PathBuf::from); - target.cxx = cfg.cxx.map(PathBuf::from); - target.ar = cfg.ar.map(PathBuf::from); - target.ranlib = cfg.ranlib.map(PathBuf::from); - target.linker = cfg.linker.map(PathBuf::from); - target.crt_static = cfg.crt_static; - target.musl_root = cfg.musl_root.map(PathBuf::from); - target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); - target.wasi_root = cfg.wasi_root.map(PathBuf::from); - target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); - target.runner = cfg.runner; - target.sanitizers = cfg.sanitizers; - target.profiler = cfg.profiler; - target.rpath = cfg.rpath; - - if let Some(ref backends) = cfg.codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - target.codegen_backends = Some(backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect()); - } - - target.split_debuginfo = cfg.split_debuginfo.as_ref().map(|v| { - v.parse().unwrap_or_else(|_| { - panic!("invalid value for target.{triple}.split-debuginfo") - }) - }); - - config.target_config.insert(TargetSelection::from_user(&triple), target); - } - } - - if config.llvm_from_ci { - let triple = &config.build.triple; - let ci_llvm_bin = config.ci_llvm_root().join("bin"); - let build_target = config - .target_config - .entry(config.build) - .or_insert_with(|| Target::from_triple(triple)); - - check_ci_llvm!(build_target.llvm_config); - check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); - } - - if let Some(dist) = toml.dist { - let Dist { - sign_folder, - upload_addr, - src_tarball, - compression_formats, - compression_profile, - include_mingw_linker, - vendor, - } = dist; - config.dist_sign_folder = sign_folder.map(PathBuf::from); - config.dist_upload_addr = upload_addr; - config.dist_compression_formats = compression_formats; - set(&mut config.dist_compression_profile, compression_profile); - set(&mut config.rust_dist_src, src_tarball); - set(&mut config.dist_include_mingw_linker, include_mingw_linker); - config.dist_vendor = vendor.unwrap_or_else(|| { - // If we're building from git or tarball sources, enable it by default. - config.rust_info.is_managed_git_subrepository() - || config.rust_info.is_from_tarball() - }); - } - - if let Some(r) = rustfmt { - *config.initial_rustfmt.borrow_mut() = if r.exists() { - RustfmtState::SystemToolchain(r) - } else { - RustfmtState::Unavailable - }; - } - - // Now that we've reached the end of our configuration, infer the - // default values for all options that we haven't otherwise stored yet. - - config.llvm_tests = llvm_tests.unwrap_or(false); - config.llvm_enzyme = llvm_enzyme.unwrap_or(false); - config.llvm_offload = llvm_offload.unwrap_or(false); - config.llvm_plugins = llvm_plugins.unwrap_or(false); - config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); - - // We make `x86_64-unknown-linux-gnu` use the self-contained linker by default, so we will - // build our internal lld and use it as the default linker, by setting the `rust.lld` config - // to true by default: - // - on the `x86_64-unknown-linux-gnu` target - // - on the `dev` and `nightly` channels - // - when building our in-tree llvm (i.e. the target has not set an `llvm-config`), so that - // we're also able to build the corresponding lld - // - or when using an external llvm that's downloaded from CI, which also contains our prebuilt - // lld - // - otherwise, we'd be using an external llvm, and lld would not necessarily available and - // thus, disabled - // - similarly, lld will not be built nor used by default when explicitly asked not to, e.g. - // when the config sets `rust.lld = false` - if config.build.triple == "x86_64-unknown-linux-gnu" - && config.hosts == [config.build] - && (config.channel == "dev" || config.channel == "nightly") - { - let no_llvm_config = config - .target_config - .get(&config.build) - .is_some_and(|target_config| target_config.llvm_config.is_none()); - let enable_lld = config.llvm_from_ci || no_llvm_config; - // Prefer the config setting in case an explicit opt-out is needed. - config.lld_enabled = lld_enabled.unwrap_or(enable_lld); - } else { - set(&mut config.lld_enabled, lld_enabled); - } - - if matches!(config.lld_mode, LldMode::SelfContained) - && !config.lld_enabled - && flags.stage.unwrap_or(0) > 0 - { - panic!( - "Trying to use self-contained lld as a linker, but LLD is not being added to the sysroot. Enable it with rust.lld = true." - ); - } - - let default_std_features = BTreeSet::from([String::from("panic-unwind")]); - config.rust_std_features = std_features.unwrap_or(default_std_features); - - let default = debug == Some(true); - config.rustc_debug_assertions = rustc_debug_assertions.unwrap_or(default); - config.std_debug_assertions = std_debug_assertions.unwrap_or(config.rustc_debug_assertions); - config.rust_overflow_checks = overflow_checks.unwrap_or(default); - config.rust_overflow_checks_std = - overflow_checks_std.unwrap_or(config.rust_overflow_checks); - - config.rust_debug_logging = debug_logging.unwrap_or(config.rustc_debug_assertions); - - let with_defaults = |debuginfo_level_specific: Option<_>| { - debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { - DebuginfoLevel::Limited - } else { - DebuginfoLevel::None - }) - }; - config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); - config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); - config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); - config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); - config.optimized_compiler_builtins = - optimized_compiler_builtins.unwrap_or(config.channel != "dev"); - config.compiletest_diff_tool = compiletest_diff_tool; - - let download_rustc = config.download_rustc_commit.is_some(); - // See https://github.com/rust-lang/compiler-team/issues/326 - config.stage = match config.cmd { - Subcommand::Check { .. } => flags.stage.or(check_stage).unwrap_or(0), - // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. - Subcommand::Doc { .. } => { - flags.stage.or(doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) - } - Subcommand::Build { .. } => { - flags.stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Test { .. } | Subcommand::Miri { .. } => { - flags.stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Bench { .. } => flags.stage.or(bench_stage).unwrap_or(2), - Subcommand::Dist { .. } => flags.stage.or(dist_stage).unwrap_or(2), - Subcommand::Install { .. } => flags.stage.or(install_stage).unwrap_or(2), - Subcommand::Perf { .. } => flags.stage.unwrap_or(1), - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Subcommand::Clean { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } - | Subcommand::Vendor { .. } => flags.stage.unwrap_or(0), - }; - - // CI should always run stage 2 builds, unless it specifically states otherwise - #[cfg(not(test))] - if flags.stage.is_none() && build_helper::ci::CiEnv::is_ci() { - match config.cmd { - Subcommand::Test { .. } - | Subcommand::Miri { .. } - | Subcommand::Doc { .. } - | Subcommand::Build { .. } - | Subcommand::Bench { .. } - | Subcommand::Dist { .. } - | Subcommand::Install { .. } => { - assert_eq!( - config.stage, 2, - "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", - config.stage, - ); - } - Subcommand::Clean { .. } - | Subcommand::Check { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } - | Subcommand::Vendor { .. } - | Subcommand::Perf { .. } => {} - } - } - - config - } - - pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - #[deprecated = "use `Builder::try_run` instead where possible"] - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(|| println!("running: {cmd:?}")); - build_helper::util::try_run(cmd, self.is_verbose()) - } - - pub(crate) fn test_args(&self) -> Vec<&str> { - let mut test_args = match self.cmd { - Subcommand::Test { ref test_args, .. } - | Subcommand::Bench { ref test_args, .. } - | Subcommand::Miri { ref test_args, .. } => { - test_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - test_args.extend(self.free_args.iter().map(|s| s.as_str())); - test_args - } - - pub(crate) fn args(&self) -> Vec<&str> { - let mut args = match self.cmd { - Subcommand::Run { ref args, .. } => { - args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - args.extend(self.free_args.iter().map(|s| s.as_str())); - args - } - - /// Returns the content of the given file at a specific commit. - pub(crate) fn read_file_by_commit(&self, file: &Path, commit: &str) -> String { - assert!( - self.rust_info.is_managed_git_subrepository(), - "`Config::read_file_by_commit` is not supported in non-git sources." - ); - - let mut git = helpers::git(Some(&self.src)); - git.arg("show").arg(format!("{commit}:{}", file.to_str().unwrap())); - output(git.as_command_mut()) - } - - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. - /// Return the version it would have used for the given commit. - pub(crate) fn artifact_version_part(&self, commit: &str) -> String { - let (channel, version) = if self.rust_info.is_managed_git_subrepository() { - let channel = self - .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) - .trim() - .to_owned(); - let version = - self.read_file_by_commit(&PathBuf::from("src/version"), commit).trim().to_owned(); - (channel, version) - } else { - let channel = fs::read_to_string(self.src.join("src/ci/channel")); - let version = fs::read_to_string(self.src.join("src/version")); - match (channel, version) { - (Ok(channel), Ok(version)) => { - (channel.trim().to_owned(), version.trim().to_owned()) - } - (channel, version) => { - let src = self.src.display(); - eprintln!("ERROR: failed to determine artifact channel and/or version"); - eprintln!( - "HELP: consider using a git checkout or ensure these files are readable" - ); - if let Err(channel) = channel { - eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); - } - if let Err(version) = version { - eprintln!("reading {src}/src/version failed: {version:?}"); - } - panic!(); - } - } - }; - - match channel.as_str() { - "stable" => version, - "beta" => channel, - "nightly" => channel, - other => unreachable!("{:?} is not recognized as a valid channel", other), - } - } - - /// Try to find the relative path of `bindir`, otherwise return it in full. - pub fn bindir_relative(&self) -> &Path { - let bindir = &self.bindir; - if bindir.is_absolute() { - // Try to make it relative to the prefix. - if let Some(prefix) = &self.prefix { - if let Ok(stripped) = bindir.strip_prefix(prefix) { - return stripped; - } - } - } - bindir - } - - /// Try to find the relative path of `libdir`. - pub fn libdir_relative(&self) -> Option<&Path> { - let libdir = self.libdir.as_ref()?; - if libdir.is_relative() { - Some(libdir) - } else { - // Try to make it relative to the prefix. - libdir.strip_prefix(self.prefix.as_ref()?).ok() - } - } - - /// The absolute path to the downloaded LLVM artifacts. - pub(crate) fn ci_llvm_root(&self) -> PathBuf { - assert!(self.llvm_from_ci); - self.out.join(self.build).join("ci-llvm") - } - - /// Directory where the extracted `rustc-dev` component is stored. - pub(crate) fn ci_rustc_dir(&self) -> PathBuf { - assert!(self.download_rustc()); - self.out.join(self.build).join("ci-rustc") - } - - /// Determine whether llvm should be linked dynamically. - /// - /// If `false`, llvm should be linked statically. - /// This is computed on demand since LLVM might have to first be downloaded from CI. - pub(crate) fn llvm_link_shared(&self) -> bool { - let mut opt = self.llvm_link_shared.get(); - if opt.is_none() && self.dry_run() { - // just assume static for now - dynamic linking isn't supported on all platforms - return false; - } - - let llvm_link_shared = *opt.get_or_insert_with(|| { - if self.llvm_from_ci { - self.maybe_download_ci_llvm(); - let ci_llvm = self.ci_llvm_root(); - let link_type = t!( - std::fs::read_to_string(ci_llvm.join("link-type.txt")), - format!("CI llvm missing: {}", ci_llvm.display()) - ); - link_type == "dynamic" - } else { - // unclear how thought-through this default is, but it maintains compatibility with - // previous behavior - false - } - }); - self.llvm_link_shared.set(opt); - llvm_link_shared - } - - /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. - pub(crate) fn download_rustc(&self) -> bool { - self.download_rustc_commit().is_some() - } - - pub(crate) fn download_rustc_commit(&self) -> Option<&str> { - static DOWNLOAD_RUSTC: OnceLock> = OnceLock::new(); - if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { - // avoid trying to actually download the commit - return self.download_rustc_commit.as_deref(); - } - - DOWNLOAD_RUSTC - .get_or_init(|| match &self.download_rustc_commit { - None => None, - Some(commit) => { - self.download_ci_rustc(commit); - - // CI-rustc can't be used without CI-LLVM. If `self.llvm_from_ci` is false, it means the "if-unchanged" - // logic has detected some changes in the LLVM submodule (download-ci-llvm=false can't happen here as - // we don't allow it while parsing the configuration). - if !self.llvm_from_ci { - // This happens when LLVM submodule is updated in CI, we should disable ci-rustc without an error - // to not break CI. For non-CI environments, we should return an error. - if CiEnv::is_ci() { - println!("WARNING: LLVM submodule has changes, `download-rustc` will be disabled."); - return None; - } else { - panic!("ERROR: LLVM submodule has changes, `download-rustc` can't be used."); - } - } - - if let Some(config_path) = &self.config { - let ci_config_toml = match self.get_builder_toml("ci-rustc") { - Ok(ci_config_toml) => ci_config_toml, - Err(e) if e.to_string().contains("unknown field") => { - println!("WARNING: CI rustc has some fields that are no longer supported in bootstrap; download-rustc will be disabled."); - println!("HELP: Consider rebasing to a newer commit if available."); - return None; - }, - Err(e) => { - eprintln!("ERROR: Failed to parse CI rustc config.toml: {e}"); - exit!(2); - }, - }; - - let current_config_toml = Self::get_toml(config_path).unwrap(); - - // Check the config compatibility - // FIXME: this doesn't cover `--set` flags yet. - let res = check_incompatible_options_for_ci_rustc( - current_config_toml, - ci_config_toml, - ); - - // Primarily used by CI runners to avoid handling download-rustc incompatible - // options one by one on shell scripts. - let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE") - .is_some_and(|s| s == "1" || s == "true"); - - if disable_ci_rustc_if_incompatible && res.is_err() { - println!("WARNING: download-rustc is disabled with `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` env."); - return None; - } - - res.unwrap(); - } - - Some(commit.clone()) - } - }) - .as_deref() - } - - pub(crate) fn initial_rustfmt(&self) -> Option { - match &mut *self.initial_rustfmt.borrow_mut() { - RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), - RustfmtState::Unavailable => None, - r @ RustfmtState::LazyEvaluated => { - if self.dry_run() { - return Some(PathBuf::new()); - } - let path = self.maybe_download_rustfmt(); - *r = if let Some(p) = &path { - RustfmtState::Downloaded(p.clone()) - } else { - RustfmtState::Unavailable - }; - path - } - } - } - - /// Runs a function if verbosity is greater than 0 - pub fn verbose(&self, f: impl Fn()) { - if self.is_verbose() { - f() - } - } - - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).and_then(|t| t.sanitizers).unwrap_or(self.sanitizers) - } - - pub fn needs_sanitizer_runtime_built(&self, target: TargetSelection) -> bool { - // MSVC uses the Microsoft-provided sanitizer runtime, but all other runtimes we build. - !target.is_msvc() && self.sanitizers_enabled(target) - } - - pub fn any_sanitizers_to_build(&self) -> bool { - self.target_config - .iter() - .any(|(ts, t)| !ts.is_msvc() && t.sanitizers.unwrap_or(self.sanitizers)) - } - - pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { - match self.target_config.get(&target)?.profiler.as_ref()? { - StringOrBool::String(s) => Some(s), - StringOrBool::Bool(_) => None, - } - } - - pub fn profiler_enabled(&self, target: TargetSelection) -> bool { - self.target_config - .get(&target) - .and_then(|t| t.profiler.as_ref()) - .map(StringOrBool::is_string_or_true) - .unwrap_or(self.profiler) - } - - pub fn any_profiler_enabled(&self) -> bool { - self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) - || self.profiler - } - - pub fn rpath_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).and_then(|t| t.rpath).unwrap_or(self.rust_rpath) - } - - pub fn llvm_enabled(&self, target: TargetSelection) -> bool { - self.codegen_backends(target).contains(&"llvm".to_owned()) - } - - pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { - self.target_config - .get(&target) - .and_then(|t| t.llvm_libunwind) - .or(self.llvm_libunwind_default) - .unwrap_or(if target.contains("fuchsia") { - LlvmLibunwind::InTree - } else { - LlvmLibunwind::No - }) - } - - pub fn split_debuginfo(&self, target: TargetSelection) -> SplitDebuginfo { - self.target_config - .get(&target) - .and_then(|t| t.split_debuginfo) - .unwrap_or_else(|| SplitDebuginfo::default_for_platform(target)) - } - - /// Returns whether or not submodules should be managed by bootstrap. - pub fn submodules(&self) -> bool { - // If not specified in config, the default is to only manage - // submodules if we're currently inside a git repository. - self.submodules.unwrap_or(self.rust_info.is_managed_git_subrepository()) - } - - pub fn codegen_backends(&self, target: TargetSelection) -> &[String] { - self.target_config - .get(&target) - .and_then(|cfg| cfg.codegen_backends.as_deref()) - .unwrap_or(&self.rust_codegen_backends) - } - - pub fn default_codegen_backend(&self, target: TargetSelection) -> Option { - self.codegen_backends(target).first().cloned() - } - - pub fn git_config(&self) -> GitConfig<'_> { - GitConfig { - git_repository: &self.stage0_metadata.config.git_repository, - nightly_branch: &self.stage0_metadata.config.nightly_branch, - git_merge_commit_email: &self.stage0_metadata.config.git_merge_commit_email, - } - } - - /// Given a path to the directory of a submodule, update it. - /// - /// `relative_path` should be relative to the root of the git repository, not an absolute path. - /// - /// This *does not* update the submodule if `config.toml` explicitly says - /// not to, or if we're not in a git repository (like a plain source - /// tarball). Typically [`crate::Build::require_submodule`] should be - /// used instead to provide a nice error to the user if the submodule is - /// missing. - pub(crate) fn update_submodule(&self, relative_path: &str) { - if !self.submodules() { - return; - } - - let absolute_path = self.src.join(relative_path); - - // NOTE: The check for the empty directory is here because when running x.py the first time, - // the submodule won't be checked out. Check it out now so we can build it. - if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() - && !helpers::dir_is_empty(&absolute_path) - { - return; - } - - // Submodule updating actually happens during in the dry run mode. We need to make sure that - // all the git commands below are actually executed, because some follow-up code - // in bootstrap might depend on the submodules being checked out. Furthermore, not all - // the command executions below work with an empty output (produced during dry run). - // Therefore, all commands below are marked with `run_always()`, so that they also run in - // dry run mode. - let submodule_git = || { - let mut cmd = helpers::git(Some(&absolute_path)); - cmd.run_always(); - cmd - }; - - // Determine commit checked out in submodule. - let checked_out_hash = output(submodule_git().args(["rev-parse", "HEAD"]).as_command_mut()); - let checked_out_hash = checked_out_hash.trim_end(); - // Determine commit that the submodule *should* have. - let recorded = output( - helpers::git(Some(&self.src)) - .run_always() - .args(["ls-tree", "HEAD"]) - .arg(relative_path) - .as_command_mut(), - ); - - let actual_hash = recorded - .split_whitespace() - .nth(2) - .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - - if actual_hash == checked_out_hash { - // already checked out - return; - } - - println!("Updating submodule {relative_path}"); - self.check_run( - helpers::git(Some(&self.src)) - .run_always() - .args(["submodule", "-q", "sync"]) - .arg(relative_path), - ); - - // Try passing `--progress` to start, then run git again without if that fails. - let update = |progress: bool| { - // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, - // even though that has no relation to the upstream for the submodule. - let current_branch = output_result( - helpers::git(Some(&self.src)) - .allow_failure() - .run_always() - .args(["symbolic-ref", "--short", "HEAD"]) - .as_command_mut(), - ) - .map(|b| b.trim().to_owned()); - - let mut git = helpers::git(Some(&self.src)).allow_failure(); - git.run_always(); - if let Ok(branch) = current_branch { - // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. - // This syntax isn't accepted by `branch.{branch}`. Strip it. - let branch = branch.strip_prefix("heads/").unwrap_or(&branch); - git.arg("-c").arg(format!("branch.{branch}.remote=origin")); - } - git.args(["submodule", "update", "--init", "--recursive", "--depth=1"]); - if progress { - git.arg("--progress"); - } - git.arg(relative_path); - git - }; - if !self.check_run(&mut update(true)) { - self.check_run(&mut update(false)); - } - - // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). - // diff-index reports the modifications through the exit status - let has_local_modifications = !self.check_run(submodule_git().allow_failure().args([ - "diff-index", - "--quiet", - "HEAD", - ])); - if has_local_modifications { - self.check_run(submodule_git().args(["stash", "push"])); - } - - self.check_run(submodule_git().args(["reset", "-q", "--hard"])); - self.check_run(submodule_git().args(["clean", "-qdfx"])); - - if has_local_modifications { - self.check_run(submodule_git().args(["stash", "pop"])); - } - } - - #[cfg(feature = "bootstrap-self-test")] - pub fn check_stage0_version(&self, _program_path: &Path, _component_name: &'static str) {} - - /// check rustc/cargo version is same or lower with 1 apart from the building one - #[cfg(not(feature = "bootstrap-self-test"))] - pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { - use build_helper::util::fail; - - if self.dry_run() { - return; - } - - let stage0_output = output(Command::new(program_path).arg("--version")); - let mut stage0_output = stage0_output.lines().next().unwrap().split(' '); - - let stage0_name = stage0_output.next().unwrap(); - if stage0_name != component_name { - fail(&format!( - "Expected to find {component_name} at {} but it claims to be {stage0_name}", - program_path.display() - )); - } - - let stage0_version = - semver::Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) - .unwrap(); - let source_version = semver::Version::parse( - fs::read_to_string(self.src.join("src/version")).unwrap().trim(), - ) - .unwrap(); - if !(source_version == stage0_version - || (source_version.major == stage0_version.major - && (source_version.minor == stage0_version.minor - || source_version.minor == stage0_version.minor + 1))) - { - let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); - fail(&format!( - "Unexpected {component_name} version: {stage0_version}, we should use {prev_version}/{source_version} to build source with {source_version}" - )); - } - } - - /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit( - &self, - download_rustc: Option, - llvm_assertions: bool, - ) -> Option { - if !is_download_ci_available(&self.build.triple, llvm_assertions) { - return None; - } - - // If `download-rustc` is not set, default to rebuilding. - let if_unchanged = match download_rustc { - None => self.rust_info.is_managed_git_subrepository(), - Some(StringOrBool::Bool(false)) => return None, - Some(StringOrBool::Bool(true)) => false, - Some(StringOrBool::String(s)) if s == "if-unchanged" => { - if !self.rust_info.is_managed_git_subrepository() { - println!( - "ERROR: `download-rustc=if-unchanged` is only compatible with Git managed sources." - ); - crate::exit!(1); - } - - true - } - Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {other}") - } - }; - - // RUSTC_IF_UNCHANGED_ALLOWED_PATHS - let mut allowed_paths = RUSTC_IF_UNCHANGED_ALLOWED_PATHS.to_vec(); - - // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, allow - // these changes to speed up the build process for library developers. This provides consistent - // functionality for library developers between `download-rustc=true` and `download-rustc="if-unchanged"` - // options. - if !CiEnv::is_ci() { - allowed_paths.push(":!library"); - } - - let commit = if self.rust_info.is_managed_git_subrepository() { - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - match self.last_modified_commit(&allowed_paths, "download-rustc", if_unchanged) { - Some(commit) => commit, - None => { - if if_unchanged { - return None; - } - println!("ERROR: could not find commit hash for downloading rustc"); - println!("HELP: maybe your repository history is too shallow?"); - println!("HELP: consider setting `rust.download-rustc=false` in config.toml"); - println!("HELP: or fetch enough history to include one upstream commit"); - crate::exit!(1); - } - } - } else { - channel::read_commit_info_file(&self.src) - .map(|info| info.sha.trim().to_owned()) - .expect("git-commit-info is missing in the project root") - }; - - if CiEnv::is_ci() && { - let head_sha = - output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut()); - let head_sha = head_sha.trim(); - commit == head_sha - } { - eprintln!("CI rustc commit matches with HEAD and we are in CI."); - eprintln!( - "`rustc.download-ci` functionality will be skipped as artifacts are not available." - ); - return None; - } - - Some(commit) - } - - fn parse_download_ci_llvm( - &self, - download_ci_llvm: Option, - asserts: bool, - ) -> bool { - let download_ci_llvm = download_ci_llvm.unwrap_or(StringOrBool::Bool(true)); - - let if_unchanged = || { - if self.rust_info.is_from_tarball() { - // Git is needed for running "if-unchanged" logic. - println!("ERROR: 'if-unchanged' is only compatible with Git managed sources."); - crate::exit!(1); - } - - // Fetching the LLVM submodule is unnecessary for self-tests. - #[cfg(not(feature = "bootstrap-self-test"))] - self.update_submodule("src/llvm-project"); - - // Check for untracked changes in `src/llvm-project`. - let has_changes = self - .last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true) - .is_none(); - - // Return false if there are untracked changes, otherwise check if CI LLVM is available. - if has_changes { false } else { llvm::is_ci_llvm_available(self, asserts) } - }; - - match download_ci_llvm { - StringOrBool::Bool(b) => { - if !b && self.download_rustc_commit.is_some() { - panic!( - "`llvm.download-ci-llvm` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`." - ); - } - - // If download-ci-llvm=true we also want to check that CI llvm is available - b && llvm::is_ci_llvm_available(self, asserts) - } - StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(), - StringOrBool::String(other) => { - panic!("unrecognized option for download-ci-llvm: {:?}", other) - } - } - } - - /// Returns the last commit in which any of `modified_paths` were changed, - /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. - pub fn last_modified_commit( - &self, - modified_paths: &[&str], - option_name: &str, - if_unchanged: bool, - ) -> Option { - assert!( - self.rust_info.is_managed_git_subrepository(), - "Can't run `Config::last_modified_commit` on a non-git source." - ); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading components from CI"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `{option_name}`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::exit!(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let mut git = helpers::git(Some(&self.src)); - git.args(["diff-index", "--quiet", &commit, "--"]).args(modified_paths); - - let has_changes = !t!(git.as_command_mut().status()).success(); - if has_changes { - if if_unchanged { - if self.is_verbose() { - println!( - "warning: saw changes to one of {modified_paths:?} since {commit}; \ - ignoring `{option_name}`" - ); - } - return None; - } - println!( - "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" - ); - } - - Some(commit.to_string()) - } -} - -/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. -/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. -#[cfg(not(feature = "bootstrap-self-test"))] -pub(crate) fn check_incompatible_options_for_ci_llvm( - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; - } - - macro_rules! warn { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let (Some(current_llvm_config), Some(ci_llvm_config)) = - (current_config_toml.llvm, ci_config_toml.llvm) - else { - return Ok(()); - }; - - let Llvm { - optimize, - thin_lto, - release_debuginfo, - assertions: _, - tests: _, - plugins, - ccache: _, - static_libstdcpp: _, - libzstd, - ninja: _, - targets, - experimental_targets, - link_jobs: _, - link_shared: _, - version_suffix, - clang_cl, - cflags, - cxxflags, - ldflags, - use_libcxx, - use_linker, - allow_old_toolchain, - offload, - polly, - clang, - enable_warnings, - download_ci_llvm: _, - build_config, - enzyme, - enable_projects: _, - } = ci_llvm_config; - - err!(current_llvm_config.optimize, optimize); - err!(current_llvm_config.thin_lto, thin_lto); - err!(current_llvm_config.release_debuginfo, release_debuginfo); - err!(current_llvm_config.libzstd, libzstd); - err!(current_llvm_config.targets, targets); - err!(current_llvm_config.experimental_targets, experimental_targets); - err!(current_llvm_config.clang_cl, clang_cl); - err!(current_llvm_config.version_suffix, version_suffix); - err!(current_llvm_config.cflags, cflags); - err!(current_llvm_config.cxxflags, cxxflags); - err!(current_llvm_config.ldflags, ldflags); - err!(current_llvm_config.use_libcxx, use_libcxx); - err!(current_llvm_config.use_linker, use_linker); - err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); - err!(current_llvm_config.offload, offload); - err!(current_llvm_config.polly, polly); - err!(current_llvm_config.clang, clang); - err!(current_llvm_config.build_config, build_config); - err!(current_llvm_config.plugins, plugins); - err!(current_llvm_config.enzyme, enzyme); - - warn!(current_llvm_config.enable_warnings, enable_warnings); - - Ok(()) -} - -/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. -/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. -fn check_incompatible_options_for_ci_rustc( - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `rust.{}` is incompatible with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; - } - - macro_rules! warn { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `rust.{}` has no effect with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let (Some(current_rust_config), Some(ci_rust_config)) = - (current_config_toml.rust, ci_config_toml.rust) - else { - return Ok(()); - }; - - let Rust { - // Following options are the CI rustc incompatible ones. - optimize, - randomize_layout, - debug_logging, - debuginfo_level_rustc, - llvm_tools, - llvm_bitcode_linker, - lto, - stack_protector, - strip, - lld_mode, - jemalloc, - rpath, - channel, - description, - incremental, - default_linker, - std_features, - - // Rest of the options can simply be ignored. - debug: _, - codegen_units: _, - codegen_units_std: _, - rustc_debug_assertions: _, - std_debug_assertions: _, - overflow_checks: _, - overflow_checks_std: _, - debuginfo_level: _, - debuginfo_level_std: _, - debuginfo_level_tools: _, - debuginfo_level_tests: _, - backtrace: _, - parallel_compiler: _, - musl_root: _, - verbose_tests: _, - optimize_tests: _, - codegen_tests: _, - omit_git_hash: _, - dist_src: _, - save_toolstates: _, - codegen_backends: _, - lld: _, - deny_warnings: _, - backtrace_on_ice: _, - verify_llvm_ir: _, - thin_lto_import_instr_limit: _, - remap_debuginfo: _, - test_compare_mode: _, - llvm_libunwind: _, - control_flow_guard: _, - ehcont_guard: _, - new_symbol_mangling: _, - profile_generate: _, - profile_use: _, - download_rustc: _, - validate_mir_opts: _, - frame_pointers: _, - } = ci_rust_config; - - // There are two kinds of checks for CI rustc incompatible options: - // 1. Checking an option that may change the compiler behaviour/output. - // 2. Checking an option that have no effect on the compiler behaviour/output. - // - // If the option belongs to the first category, we call `err` macro for a hard error; - // otherwise, we just print a warning with `warn` macro. - - err!(current_rust_config.optimize, optimize); - err!(current_rust_config.randomize_layout, randomize_layout); - err!(current_rust_config.debug_logging, debug_logging); - err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc); - err!(current_rust_config.rpath, rpath); - err!(current_rust_config.strip, strip); - err!(current_rust_config.lld_mode, lld_mode); - err!(current_rust_config.llvm_tools, llvm_tools); - err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker); - err!(current_rust_config.jemalloc, jemalloc); - err!(current_rust_config.default_linker, default_linker); - err!(current_rust_config.stack_protector, stack_protector); - err!(current_rust_config.lto, lto); - err!(current_rust_config.std_features, std_features); - - warn!(current_rust_config.channel, channel); - warn!(current_rust_config.description, description); - warn!(current_rust_config.incremental, incremental); - - Ok(()) -} - -fn set(field: &mut T, val: Option) { - if let Some(v) = val { - *field = v; - } -} - -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, - } -} +pub use target_selection; + + + // use build_helper::ci::CiEnv; + // use build_helper::exit; + // use build_helper::git::get_closest_merge_commit; + // use build_helper::git::GitConfig; + // use build_helper::git::output_result; + // use cc::Build; + // use clap::builder::styling::Color; + // use clap::Command; + // use clap::Subcommand; + // use clap::ValueEnum; + // use cmake::Config; + // use crate::BTreeSet; + // use crate::Build; + // use crate::Cell; + // use crate::Command; + // use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; + // use crate::core::build_steps::llvm; + // use crate::core::build_steps::llvm::Llvm; + // use crate::core::build_steps::setup::Profile; + // use crate::core::build_steps::setup::Profile::Dist; + // use crate::core::build_steps::tool::LibcxxVersion::Llvm; + +use changeid::ChangeIdWrapper; +use ciconfig::CiConfig; +use color::Color; +use config_base::Config; +use debug_info_level::DebuginfoLevel; +use dry_run::BUILDER_CONFIG_FILENAME; +use rustclto::RustcLto; +use rustfmt::RustfmtState; +use rust_optimize::RustOptimize; +use splitdebuginfo::SplitDebuginfo; +use stringorbool::StringOrBool; +use subcommand::get_completion; +use subcommand::Subcommand; +use subcommand::Subcommand::Build; +use subcommand::Subcommand::Dist; +use subcommand::Subcommand::Install; +use tomlconfig::TomlConfig; +use warnings::Warnings; +use bootstrap_config_utils::default_opts; +use bootstrap_config_utils::get_builder_toml; +use bootstrap_config_utils::get_toml; + +// use crate::core::download::is_download_ci_available; + // use crate::define_config; + // use crate::Display; + // use crate::DocTests; +pub use dry_run::*; + // use crate::env; + // use crate::exe; + // use crate::exit; + // use crate::Flags; + // use crate::fs; + // use crate::GitInfo; + // use crate::GitRepo::Llvm; + // use crate::HashMap; + // use crate::HashSet; + // use crate::helpers; + // use crate::Kind; + // use crate::Kind::Build; + // use crate::Kind::Dist; + // use crate::Kind::Install; +pub use lld_mode::*; + // use crate::LlvmLibunwind; + // use crate::OnceLock; + // use crate::output; + // use crate::Path; + // use crate::PathBuf; + // use crate::RefCell; + // use crate::str::FromStr; + // use crate::t; + // use crate::Target; +pub use target_selection::TargetSelection; + // use crate::utils::cache::Interned; + // use crate::utils::cache::INTERNER; + // use crate::utils::channel; + // use crate::utils::shared_helpers::exe; + // use crate::utils::tarball::OverlayKind::Llvm; + // use crate::utils::tarball::OverlayKind::Rust; + // use serde_derive::Deserialize; + // use serde::Deserialize; + // use serde::Deserializer; + // use std::cell::Cell; + // use std::cell::RefCell; + // use std::cmp; + // use std::collections::BTreeSet; + // use std::collections::HashMap; + // use std::collections::HashSet; + // use std::env; + // use std::fmt; + // use std::fmt::Display; + // use std::fs; + // use std::path::absolute; + // use std::path::Path; + // use std::path::PathBuf; + // use std::process::Command; + // use std::str::FromStr; + // use std::sync::OnceLock; + // use termcolor::Color; diff --git a/standalonex/src/bootstrap/src/core/config/config_base.rs b/standalonex/src/bootstrap/src/core/config/config_base.rs new file mode 100644 index 00000000..0624b7ee --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/config_base.rs @@ -0,0 +1,241 @@ +use crate::prelude::*; +use std::process::Command; +use std::env; +use anyhow::{Context, Result}; + +/// Global configuration for the entire build and/or bootstrap. +/// +/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. +/// +/// Note that this structure is not decoded directly into, but rather it is +/// filled out from the decoded forms of the structs below. For documentation +/// each field, see the corresponding fields in +/// `config.example.toml`. +#[derive(Default, Clone)] +pub struct Config { + pub change_id: Option, + pub bypass_bootstrap_lock: bool, + pub ccache: Option, + /// Call Build::ninja() instead of this. + pub ninja_in_file: bool, + pub verbose: usize, + pub submodules: Option, + pub compiler_docs: bool, + pub library_docs_private_items: bool, + pub docs_minification: bool, + pub docs: bool, + pub locked_deps: bool, + pub vendor: bool, + pub target_config: HashMap, + pub full_bootstrap: bool, + pub bootstrap_cache_path: Option, + pub extended: bool, + pub tools: Option>, + pub sanitizers: bool, + pub profiler: bool, + pub omit_git_hash: bool, + pub skip: Vec, + pub include_default_paths: bool, + pub rustc_error_format: Option, + pub json_output: bool, + pub test_compare_mode: bool, + pub color: Color, + pub patch_binaries_for_nix: Option, + pub stage0_path: Option, + pub stage0_metadata: build_helper::stage0_parser::Stage0, + pub android_ndk: Option, + /// Whether to use the `c` feature of the `compiler_builtins` crate. + pub optimized_compiler_builtins: bool, + + pub stdout_is_tty: bool, + pub stderr_is_tty: bool, + + pub on_fail: Option, + pub stage: u32, + pub keep_stage: Vec, + pub keep_stage_std: Vec, + pub src: PathBuf, + /// defaults to `config.toml` + pub config: Option, + pub nixpkgs_path: Option, + pub rust_overlay_path: Option, + pub rust_bootstrap_nix_path: Option, + pub configuration_nix_path: Option, + pub rust_src_flake_path: Option, + pub rustc_versions: Vec, + pub cargo_versions: Vec, + pub jobs: Option, + pub cmd: Subcommand, + pub incremental: bool, + pub dry_run: DryRun, + pub dump_bootstrap_shims: bool, + /// Arguments appearing after `--` to be forwarded to tools, + /// e.g. `--fix-broken` or test arguments. + pub free_args: Vec, + + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option, + #[cfg(test)] + pub download_rustc_commit: Option, + + pub deny_warnings: bool, + pub backtrace_on_ice: bool, + + // llvm codegen options + pub llvm_assertions: bool, + pub llvm_tests: bool, + pub llvm_enzyme: bool, + pub llvm_offload: bool, + pub llvm_plugins: bool, + pub llvm_optimize: bool, + pub llvm_thin_lto: bool, + pub llvm_release_debuginfo: bool, + pub llvm_static_stdcpp: bool, + pub llvm_libzstd: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell>, + #[cfg(test)] + pub llvm_link_shared: Cell>, + pub llvm_clang_cl: Option, + pub llvm_targets: Option, + pub llvm_experimental_targets: Option, + pub llvm_link_jobs: Option, + pub llvm_version_suffix: Option, + pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_enable_warnings: bool, + pub llvm_from_ci: bool, + pub llvm_build_config: HashMap, + pub llvm_enable_projects: Option, + + pub lld_mode: LldMode, + pub lld_enabled: bool, + pub llvm_tools_enabled: bool, + pub llvm_bitcode_linker_enabled: bool, + + pub llvm_cflags: Option, + pub llvm_cxxflags: Option, + pub llvm_ldflags: Option, + pub llvm_use_libcxx: bool, + + // rust codegen options + pub rust_optimize: RustOptimize, + pub rust_codegen_units: Option, + pub rust_codegen_units_std: Option, + + pub rustc_debug_assertions: bool, + pub std_debug_assertions: bool, + + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: DebuginfoLevel, + pub rust_debuginfo_level_std: DebuginfoLevel, + pub rust_debuginfo_level_tools: DebuginfoLevel, + pub rust_debuginfo_level_tests: DebuginfoLevel, + pub rust_rpath: bool, + pub rust_strip: bool, + pub rust_frame_pointers: bool, + pub rust_stack_protector: Option, + pub rustc_default_linker: Option, + pub rust_optimize_tests: bool, + pub rust_dist_src: bool, + pub rust_codegen_backends: Vec, + pub rust_verify_llvm_ir: bool, + pub rust_thin_lto_import_instr_limit: Option, + pub rust_randomize_layout: bool, + pub rust_remap_debuginfo: bool, + pub rust_new_symbol_mangling: Option, + pub rust_profile_use: Option, + pub rust_profile_generate: Option, + pub rust_lto: RustcLto, + pub rust_validate_mir_opts: Option, + pub rust_std_features: BTreeSet, + pub llvm_profile_use: Option, + pub llvm_profile_generate: bool, + pub llvm_libunwind_default: Option, + pub enable_bolt_settings: bool, + + pub reproducible_artifacts: Vec, + + pub build: TargetSelection, + pub hosts: Vec, + pub targets: Vec, + pub local_rebuild: bool, + pub jemalloc: bool, + pub control_flow_guard: bool, + pub ehcont_guard: bool, + + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_compression_formats: Option>, + pub dist_compression_profile: String, + pub dist_include_mingw_linker: bool, + pub dist_vendor: bool, + + // libstd features + pub backtrace: bool, // support for RUST_BACKTRACE + + // misc + pub low_priority: bool, + pub channel: String, + pub description: Option, + pub verbose_tests: bool, + pub save_toolstates: Option, + pub print_step_timings: bool, + pub print_step_rusage: bool, + + // Fallback musl-root for all targets + pub musl_root: Option, + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: PathBuf, + pub libdir: Option, + pub mandir: Option, + pub codegen_tests: bool, + pub nodejs: Option, + pub npm: Option, + pub gdb: Option, + pub lldb: Option, + pub python: Option, + pub reuse: Option, + pub cargo_native_static: bool, + pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, + + pub cargo_info: channel::GitInfo, + pub rust_analyzer_info: channel::GitInfo, + pub clippy_info: channel::GitInfo, + pub miri_info: channel::GitInfo, + pub rustfmt_info: channel::GitInfo, + pub enzyme_info: channel::GitInfo, + pub in_tree_llvm_info: channel::GitInfo, + pub in_tree_gcc_info: channel::GitInfo, + + // These are either the stage0 downloaded binaries or the locally installed ones. + pub initial_cargo: PathBuf, + pub initial_rustc: PathBuf, + pub initial_cargo_clippy: Option, + + #[cfg(not(test))] + initial_rustfmt: RefCell, + #[cfg(test)] + pub initial_rustfmt: RefCell, + + pub ci: CiConfig, + + /// The paths to work with. For example: with `./x check foo bar` we get + /// `paths=["foo", "bar"]`. + pub paths: Vec, + + /// Command for visual diff display, e.g. `diff-tool --color=always`. + pub compiletest_diff_tool: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config/config_ci.rs b/standalonex/src/bootstrap/src/core/config/config_ci.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config/config_part2.rs b/standalonex/src/bootstrap/src/core/config/config_part2.rs new file mode 100644 index 00000000..f143513d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/config_part2.rs @@ -0,0 +1,148 @@ +use crate::prelude::*; +/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. +/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. +pub fn check_incompatible_options_for_ci_rustc( + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `rust.{}` is incompatible with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `rust.{}` has no effect with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let (Some(current_rust_config), Some(ci_rust_config)) = + (current_config_toml.rust, ci_config_toml.rust) + else { + return Ok(()); + }; + + let Rust { + // Following options are the CI rustc incompatible ones. + optimize, + randomize_layout, + debug_logging, + debuginfo_level_rustc, + llvm_tools, + llvm_bitcode_linker, + lto, + stack_protector, + strip, + lld_mode, + jemalloc, + rpath, + channel, + description, + incremental, + default_linker, + std_features, + + // Rest of the options can simply be ignored. + debug: _, + codegen_units: _, + codegen_units_std: _, + rustc_debug_assertions: _, + std_debug_assertions: _, + overflow_checks: _, + overflow_checks_std: _, + debuginfo_level: _, + debuginfo_level_std: _, + debuginfo_level_tools: _, + debuginfo_level_tests: _, + backtrace: _, + parallel_compiler: _, + musl_root: _, + verbose_tests: _, + optimize_tests: _, + codegen_tests: _, + omit_git_hash: _, + dist_src: _, + save_toolstates: _, + codegen_backends: _, + lld: _, + deny_warnings: _, + backtrace_on_ice: _, + verify_llvm_ir: _, + thin_lto_import_instr_limit: _, + remap_debuginfo: _, + test_compare_mode: _, + llvm_libunwind: _, + control_flow_guard: _, + ehcont_guard: _, + new_symbol_mangling: _, + profile_generate: _, + profile_use: _, + download_rustc: _, + validate_mir_opts: _, + frame_pointers: _, + } = ci_rust_config; + + // There are two kinds of checks for CI rustc incompatible options: + // 1. Checking an option that may change the compiler behaviour/output. + // 2. Checking an option that have no effect on the compiler behaviour/output. + // + // If the option belongs to the first category, we call `err` macro for a hard error; + // otherwise, we just print a warning with `warn` macro. + + err!(current_rust_config.optimize, optimize); + err!(current_rust_config.randomize_layout, randomize_layout); + err!(current_rust_config.debug_logging, debug_logging); + err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc); + err!(current_rust_config.rpath, rpath); + err!(current_rust_config.strip, strip); + err!(current_rust_config.lld_mode, lld_mode); + err!(current_rust_config.llvm_tools, llvm_tools); + err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker); + err!(current_rust_config.jemalloc, jemalloc); + err!(current_rust_config.default_linker, default_linker); + err!(current_rust_config.stack_protector, stack_protector); + err!(current_rust_config.lto, lto); + err!(current_rust_config.std_features, std_features); + + warn!(current_rust_config.channel, channel); + warn!(current_rust_config.description, description); + warn!(current_rust_config.incremental, incremental); + + Ok(()) +} + +pub fn set(field: &mut T, val: Option) { + if let Some(v) = val { + *field = v; + } +} + +pub fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, + } +} diff --git a/standalonex/src/bootstrap/src/core/config/config_part3.rs b/standalonex/src/bootstrap/src/core/config/config_part3.rs new file mode 100644 index 00000000..fd757152 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/config_part3.rs @@ -0,0 +1,107 @@ +use crate::prelude::*; +/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. +/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. +#[cfg(not(feature = "bootstrap-self-test"))] +pub(crate) fn check_incompatible_options_for_ci_llvm( + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let (Some(current_llvm_config), Some(ci_llvm_config)) = + (current_config_toml.llvm, ci_config_toml.llvm) + else { + return Ok(()); + }; + + let Llvm { + optimize, + thin_lto, + release_debuginfo, + assertions: _, + tests: _, + plugins, + ccache: _, + static_libstdcpp: _, + libzstd, + ninja: _, + targets, + experimental_targets, + link_jobs: _, + link_shared: _, + version_suffix, + clang_cl, + cflags, + cxxflags, + ldflags, + use_libcxx, + use_linker, + allow_old_toolchain, + offload, + polly, + clang, + enable_warnings, + download_ci_llvm: _, + build_config, + enzyme, + enable_projects: _, + } = ci_llvm_config; + + err!(current_llvm_config.optimize, optimize); + err!(current_llvm_config.thin_lto, thin_lto); + err!(current_llvm_config.release_debuginfo, release_debuginfo); + err!(current_llvm_config.libzstd, libzstd); + err!(current_llvm_config.targets, targets); + err!(current_llvm_config.experimental_targets, experimental_targets); + err!(current_llvm_config.clang_cl, clang_cl); + err!(current_llvm_config.version_suffix, version_suffix); + err!(current_llvm_config.cflags, cflags); + err!(current_llvm_config.cxxflags, cxxflags); + err!(current_llvm_config.ldflags, ldflags); + err!(current_llvm_config.use_libcxx, use_libcxx); + err!(current_llvm_config.use_linker, use_linker); + err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); + err!(current_llvm_config.offload, offload); + err!(current_llvm_config.polly, polly); + err!(current_llvm_config.clang, clang); + err!(current_llvm_config.build_config, build_config); + err!(current_llvm_config.plugins, plugins); + err!(current_llvm_config.enzyme, enzyme); + + warn!(current_llvm_config.enable_warnings, enable_warnings); + + Ok(()) +} + diff --git a/standalonex/src/src/core/config/config.rs b/standalonex/src/bootstrap/src/core/config/config_part4.rs similarity index 53% rename from standalonex/src/src/core/config/config.rs rename to standalonex/src/bootstrap/src/core/config/config_part4.rs index d327a465..23bbc4bd 100644 --- a/standalonex/src/src/core/config/config.rs +++ b/standalonex/src/bootstrap/src/core/config/config_part4.rs @@ -1,1311 +1,10 @@ -//! Serialized configuration of a build. -//! -//! This module implements parsing `config.toml` configuration files to tweak -//! how the build runs. - -use std::cell::{Cell, RefCell}; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fmt::{self, Display}; -use std::io::IsTerminal; -use std::path::{Path, PathBuf, absolute}; -use std::process::Command; -use std::str::FromStr; -use std::sync::OnceLock; -use std::{cmp, env, fs}; - -use build_helper::ci::CiEnv; -use build_helper::exit; -use build_helper::git::{GitConfig, get_closest_merge_commit, output_result}; -use serde::{Deserialize, Deserializer}; -use serde_derive::Deserialize; - -use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; -use crate::core::build_steps::llvm; -pub use crate::core::config::flags::Subcommand; -use crate::core::config::flags::{Color, Flags, Warnings}; -use crate::core::download::is_download_ci_available; -use crate::utils::cache::{INTERNER, Interned}; -use crate::utils::channel::{self, GitInfo}; -use crate::utils::helpers::{self, exe, output, t}; - -/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. -/// This means they can be modified and changes to these paths should never trigger a compiler build -/// when "if-unchanged" is set. -/// -/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during -/// the diff check. -/// -/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build -/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. -/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the -/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. -#[rustfmt::skip] // We don't want rustfmt to oneline this list -pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ - ":!src/tools", - ":!tests", - ":!triagebot.toml", -]; - -macro_rules! check_ci_llvm { - ($name:expr) => { - assert!( - $name.is_none(), - "setting {} is incompatible with download-ci-llvm.", - stringify!($name).replace("_", "-") - ); - }; -} - -/// This file is embedded in the overlay directory of the tarball sources. It is -/// useful in scenarios where developers want to see how the tarball sources were -/// generated. -/// -/// We also use this file to compare the host's config.toml against the CI rustc builder -/// configuration to detect any incompatible options. -pub(crate) const BUILDER_CONFIG_FILENAME: &str = "builder-config"; - -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - -#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] -pub enum DebuginfoLevel { - #[default] - None, - LineDirectivesOnly, - LineTablesOnly, - Limited, - Full, -} - -// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only -// deserializes i64, and derive() only generates visit_u64 -impl<'de> Deserialize<'de> for DebuginfoLevel { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - use serde::de::Error; - - Ok(match Deserialize::deserialize(deserializer)? { - StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, - StringOrInt::Int(0) => DebuginfoLevel::None, - StringOrInt::String(s) if s == "line-directives-only" => { - DebuginfoLevel::LineDirectivesOnly - } - StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, - StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, - StringOrInt::Int(1) => DebuginfoLevel::Limited, - StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, - StringOrInt::Int(2) => DebuginfoLevel::Full, - StringOrInt::Int(n) => { - let other = serde::de::Unexpected::Signed(n); - return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); - } - StringOrInt::String(s) => { - let other = serde::de::Unexpected::Str(&s); - return Err(D::Error::invalid_value( - other, - &"expected none, line-tables-only, limited, or full", - )); - } - }) - } -} - -/// Suitable for passing to `-C debuginfo` -impl Display for DebuginfoLevel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use DebuginfoLevel::*; - f.write_str(match self { - None => "0", - LineDirectivesOnly => "line-directives-only", - LineTablesOnly => "line-tables-only", - Limited => "1", - Full => "2", - }) - } -} - -/// LLD in bootstrap works like this: -/// - Self-contained lld: use `rust-lld` from the compiler's sysroot -/// - External: use an external `lld` binary -/// -/// It is configured depending on the target: -/// 1) Everything except MSVC -/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` -/// - External: `-Clinker-flavor=gnu-lld-cc` -/// 2) MSVC -/// - Self-contained: `-Clinker=` -/// - External: `-Clinker=lld` -#[derive(Copy, Clone, Default, Debug, PartialEq)] -pub enum LldMode { - /// Do not use LLD - #[default] - Unused, - /// Use `rust-lld` from the compiler's sysroot - SelfContained, - /// Use an externally provided `lld` binary. - /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has - /// to be in $PATH. - External, -} - -impl LldMode { - pub fn is_used(&self) -> bool { - match self { - LldMode::SelfContained | LldMode::External => true, - LldMode::Unused => false, - } - } -} - -/// Global configuration for the entire build and/or bootstrap. -/// -/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. -/// -/// Note that this structure is not decoded directly into, but rather it is -/// filled out from the decoded forms of the structs below. For documentation -/// each field, see the corresponding fields in -/// `config.example.toml`. -#[derive(Default, Clone)] -pub struct Config { - pub change_id: Option, - pub bypass_bootstrap_lock: bool, - pub ccache: Option, - /// Call Build::ninja() instead of this. - pub ninja_in_file: bool, - pub verbose: usize, - pub submodules: Option, - pub compiler_docs: bool, - pub library_docs_private_items: bool, - pub docs_minification: bool, - pub docs: bool, - pub locked_deps: bool, - pub vendor: bool, - pub target_config: HashMap, - pub full_bootstrap: bool, - pub bootstrap_cache_path: Option, - pub extended: bool, - pub tools: Option>, - pub sanitizers: bool, - pub profiler: bool, - pub omit_git_hash: bool, - pub skip: Vec, - pub include_default_paths: bool, - pub rustc_error_format: Option, - pub json_output: bool, - pub test_compare_mode: bool, - pub color: Color, - pub patch_binaries_for_nix: Option, - pub stage0_metadata: build_helper::stage0_parser::Stage0, - pub android_ndk: Option, - /// Whether to use the `c` feature of the `compiler_builtins` crate. - pub optimized_compiler_builtins: bool, - - pub stdout_is_tty: bool, - pub stderr_is_tty: bool, - - pub on_fail: Option, - pub stage: u32, - pub keep_stage: Vec, - pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, - pub jobs: Option, - pub cmd: Subcommand, - pub incremental: bool, - pub dry_run: DryRun, - pub dump_bootstrap_shims: bool, - /// Arguments appearing after `--` to be forwarded to tools, - /// e.g. `--fix-broken` or test arguments. - pub free_args: Vec, - - /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. - #[cfg(not(test))] - download_rustc_commit: Option, - #[cfg(test)] - pub download_rustc_commit: Option, - - pub deny_warnings: bool, - pub backtrace_on_ice: bool, - - // llvm codegen options - pub llvm_assertions: bool, - pub llvm_tests: bool, - pub llvm_enzyme: bool, - pub llvm_offload: bool, - pub llvm_plugins: bool, - pub llvm_optimize: bool, - pub llvm_thin_lto: bool, - pub llvm_release_debuginfo: bool, - pub llvm_static_stdcpp: bool, - pub llvm_libzstd: bool, - /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. - #[cfg(not(test))] - llvm_link_shared: Cell>, - #[cfg(test)] - pub llvm_link_shared: Cell>, - pub llvm_clang_cl: Option, - pub llvm_targets: Option, - pub llvm_experimental_targets: Option, - pub llvm_link_jobs: Option, - pub llvm_version_suffix: Option, - pub llvm_use_linker: Option, - pub llvm_allow_old_toolchain: bool, - pub llvm_polly: bool, - pub llvm_clang: bool, - pub llvm_enable_warnings: bool, - pub llvm_from_ci: bool, - pub llvm_build_config: HashMap, - pub llvm_enable_projects: Option, - - pub lld_mode: LldMode, - pub lld_enabled: bool, - pub llvm_tools_enabled: bool, - pub llvm_bitcode_linker_enabled: bool, - - pub llvm_cflags: Option, - pub llvm_cxxflags: Option, - pub llvm_ldflags: Option, - pub llvm_use_libcxx: bool, - - // rust codegen options - pub rust_optimize: RustOptimize, - pub rust_codegen_units: Option, - pub rust_codegen_units_std: Option, - - pub rustc_debug_assertions: bool, - pub std_debug_assertions: bool, - - pub rust_overflow_checks: bool, - pub rust_overflow_checks_std: bool, - pub rust_debug_logging: bool, - pub rust_debuginfo_level_rustc: DebuginfoLevel, - pub rust_debuginfo_level_std: DebuginfoLevel, - pub rust_debuginfo_level_tools: DebuginfoLevel, - pub rust_debuginfo_level_tests: DebuginfoLevel, - pub rust_rpath: bool, - pub rust_strip: bool, - pub rust_frame_pointers: bool, - pub rust_stack_protector: Option, - pub rustc_default_linker: Option, - pub rust_optimize_tests: bool, - pub rust_dist_src: bool, - pub rust_codegen_backends: Vec, - pub rust_verify_llvm_ir: bool, - pub rust_thin_lto_import_instr_limit: Option, - pub rust_randomize_layout: bool, - pub rust_remap_debuginfo: bool, - pub rust_new_symbol_mangling: Option, - pub rust_profile_use: Option, - pub rust_profile_generate: Option, - pub rust_lto: RustcLto, - pub rust_validate_mir_opts: Option, - pub rust_std_features: BTreeSet, - pub llvm_profile_use: Option, - pub llvm_profile_generate: bool, - pub llvm_libunwind_default: Option, - pub enable_bolt_settings: bool, - - pub reproducible_artifacts: Vec, - - pub build: TargetSelection, - pub hosts: Vec, - pub targets: Vec, - pub local_rebuild: bool, - pub jemalloc: bool, - pub control_flow_guard: bool, - pub ehcont_guard: bool, - - // dist misc - pub dist_sign_folder: Option, - pub dist_upload_addr: Option, - pub dist_compression_formats: Option>, - pub dist_compression_profile: String, - pub dist_include_mingw_linker: bool, - pub dist_vendor: bool, - - // libstd features - pub backtrace: bool, // support for RUST_BACKTRACE - - // misc - pub low_priority: bool, - pub channel: String, - pub description: Option, - pub verbose_tests: bool, - pub save_toolstates: Option, - pub print_step_timings: bool, - pub print_step_rusage: bool, - - // Fallback musl-root for all targets - pub musl_root: Option, - pub prefix: Option, - pub sysconfdir: Option, - pub datadir: Option, - pub docdir: Option, - pub bindir: PathBuf, - pub libdir: Option, - pub mandir: Option, - pub codegen_tests: bool, - pub nodejs: Option, - pub npm: Option, - pub gdb: Option, - pub lldb: Option, - pub python: Option, - pub reuse: Option, - pub cargo_native_static: bool, - pub configure_args: Vec, - pub out: PathBuf, - pub rust_info: channel::GitInfo, - - pub cargo_info: channel::GitInfo, - pub rust_analyzer_info: channel::GitInfo, - pub clippy_info: channel::GitInfo, - pub miri_info: channel::GitInfo, - pub rustfmt_info: channel::GitInfo, - pub enzyme_info: channel::GitInfo, - pub in_tree_llvm_info: channel::GitInfo, - pub in_tree_gcc_info: channel::GitInfo, - - // These are either the stage0 downloaded binaries or the locally installed ones. - pub initial_cargo: PathBuf, - pub initial_rustc: PathBuf, - pub initial_cargo_clippy: Option, - - #[cfg(not(test))] - initial_rustfmt: RefCell, - #[cfg(test)] - pub initial_rustfmt: RefCell, - - /// The paths to work with. For example: with `./x check foo bar` we get - /// `paths=["foo", "bar"]`. - pub paths: Vec, - - /// Command for visual diff display, e.g. `diff-tool --color=always`. - pub compiletest_diff_tool: Option, -} - -#[derive(Clone, Debug, Default)] -pub enum RustfmtState { - SystemToolchain(PathBuf), - Downloaded(PathBuf), - Unavailable, - #[default] - LazyEvaluated, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub enum LlvmLibunwind { - #[default] - No, - InTree, - System, -} - -impl FromStr for LlvmLibunwind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "no" => Ok(Self::No), - "in-tree" => Ok(Self::InTree), - "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), - } - } -} - -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SplitDebuginfo { - Packed, - Unpacked, - #[default] - Off, -} - -impl std::str::FromStr for SplitDebuginfo { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "packed" => Ok(SplitDebuginfo::Packed), - "unpacked" => Ok(SplitDebuginfo::Unpacked), - "off" => Ok(SplitDebuginfo::Off), - _ => Err(()), - } - } -} - -impl SplitDebuginfo { - /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `config.example.toml`. - fn default_for_platform(target: TargetSelection) -> Self { - if target.contains("apple") { - SplitDebuginfo::Unpacked - } else if target.is_windows() { - SplitDebuginfo::Packed - } else { - SplitDebuginfo::Off - } - } -} - -/// LTO mode used for compiling rustc itself. -#[derive(Default, Clone, PartialEq, Debug)] -pub enum RustcLto { - Off, - #[default] - ThinLocal, - Thin, - Fat, -} - -impl std::str::FromStr for RustcLto { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "thin-local" => Ok(RustcLto::ThinLocal), - "thin" => Ok(RustcLto::Thin), - "fat" => Ok(RustcLto::Fat), - "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {s}")), - } - } -} - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. -// Making !Copy is highly nontrivial! -pub struct TargetSelection { - pub triple: Interned, - file: Option>, - synthetic: bool, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file, synthetic: false } - } - - pub fn create_synthetic(triple: &str, file: &str) -> Self { - Self { - triple: INTERNER.intern_str(triple), - file: Some(INTERNER.intern_str(file)), - synthetic: true, - } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } - - // See src/bootstrap/synthetic_targets.rs - pub fn is_synthetic(&self) -> bool { - self.synthetic - } - - pub fn is_msvc(&self) -> bool { - self.contains("msvc") - } - - pub fn is_windows(&self) -> bool { - self.contains("windows") - } - - pub fn is_windows_gnu(&self) -> bool { - self.ends_with("windows-gnu") - } - - /// Path to the file defining the custom target, if any. - pub fn filepath(&self) -> Option<&Path> { - self.file.as_ref().map(Path::new) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({file})")?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - -// Targets are often used as directory names throughout bootstrap. -// This impl makes it more ergonomics to use them as such. -impl AsRef for TargetSelection { - fn as_ref(&self) -> &Path { - self.triple.as_ref() - } -} - -/// Per-target configuration stored in the global configuration structure. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct Target { - /// Some(path to llvm-config) if using an external LLVM. - pub llvm_config: Option, - pub llvm_has_rust_patches: Option, - /// Some(path to FileCheck) if one was specified. - pub llvm_filecheck: Option, - pub llvm_libunwind: Option, - pub cc: Option, - pub cxx: Option, - pub ar: Option, - pub ranlib: Option, - pub default_linker: Option, - pub linker: Option, - pub split_debuginfo: Option, - pub sanitizers: Option, - pub profiler: Option, - pub rpath: Option, - pub crt_static: Option, - pub musl_root: Option, - pub musl_libdir: Option, - pub wasi_root: Option, - pub qemu_rootfs: Option, - pub runner: Option, - pub no_std: bool, - pub codegen_backends: Option>, -} - -impl Target { - pub fn from_triple(triple: &str) -> Self { - let mut target: Self = Default::default(); - if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { - target.no_std = true; - } - if triple.contains("emscripten") { - target.runner = Some("node".into()); - } - target - } -} -/// Structure of the `config.toml` file that configuration is read from. -/// -/// This structure uses `Decodable` to automatically decode a TOML configuration -/// file into this format, and then this is traversed and written into the above -/// `Config` structure. -#[derive(Deserialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub(crate) struct TomlConfig { - #[serde(flatten)] - change_id: ChangeIdWrapper, - build: Option, - install: Option, - llvm: Option, - rust: Option, - target: Option>, - dist: Option, - profile: Option, -} - -/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type -/// for the "change-id" field to parse it even if other fields are invalid. This ensures -/// that if deserialization fails due to other fields, we can still provide the changelogs -/// to allow developers to potentially find the reason for the failure in the logs.. -#[derive(Deserialize, Default)] -pub(crate) struct ChangeIdWrapper { - #[serde(alias = "change-id")] - pub(crate) inner: Option, -} - -/// Describes how to handle conflicts in merging two [`TomlConfig`] -#[derive(Copy, Clone, Debug)] -enum ReplaceOpt { - /// Silently ignore a duplicated value - IgnoreDuplicate, - /// Override the current value, even if it's `Some` - Override, - /// Exit with an error on duplicate values - ErrorOnDuplicate, -} - -trait Merge { - fn merge(&mut self, other: Self, replace: ReplaceOpt); -} - -impl Merge for TomlConfig { - fn merge( - &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self, - replace: ReplaceOpt, - ) { - fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { - if let Some(new) = y { - if let Some(original) = x { - original.merge(new, replace); - } else { - *x = Some(new); - } - } - } - self.change_id.inner.merge(change_id.inner, replace); - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - - match (self.target.as_mut(), target) { - (_, None) => {} - (None, Some(target)) => self.target = Some(target), - (Some(original_target), Some(new_target)) => { - for (triple, new) in new_target { - if let Some(original) = original_target.get_mut(&triple) { - original.merge(new, replace); - } else { - original_target.insert(triple, new); - } - } - } - } - } -} - -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap. -macro_rules! define_config { - ($(#[$attr:meta])* struct $name:ident { - $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* - }) => { - $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* - } - - impl Merge for $name { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - $( - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.$field.is_none() { - self.$field = other.$field; - } - }, - ReplaceOpt::Override => { - if other.$field.is_some() { - self.$field = other.$field; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.$field.is_some() { - if self.$field.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option: `{}`", stringify!($field)); - exit!(2); - } - } else { - self.$field = other.$field; - } - } - } - } - )* - } - } - - // The following is a trimmed version of what serde_derive generates. All parts not relevant - // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of bootstrap. - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct Field; - impl<'de> serde::de::Visitor<'de> for Field { - type Value = $name; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(concat!("struct ", stringify!($name))) - } - - #[inline] - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - $(let mut $field: Option<$field_ty> = None;)* - while let Some(key) = - match serde::de::MapAccess::next_key::(&mut map) { - Ok(val) => val, - Err(err) => { - return Err(err); - } - } - { - match &*key { - $($field_key => { - if $field.is_some() { - return Err(::duplicate_field( - $field_key, - )); - } - $field = match serde::de::MapAccess::next_value::<$field_ty>( - &mut map, - ) { - Ok(val) => Some(val), - Err(err) => { - return Err(err); - } - }; - })* - key => { - return Err(serde::de::Error::unknown_field(key, FIELDS)); - } - } - } - Ok($name { $($field),* }) - } - } - const FIELDS: &'static [&'static str] = &[ - $($field_key,)* - ]; - Deserializer::deserialize_struct( - deserializer, - stringify!($name), - FIELDS, - Field, - ) - } - } - } -} - -impl Merge for Option { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.is_none() { - *self = other; - } - } - ReplaceOpt::Override => { - if other.is_some() { - *self = other; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.is_some() { - if self.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option"); - exit!(2); - } - } else { - *self = other; - } - } - } - } - } -} - -define_config! { - /// TOML representation of various global build decisions. - #[derive(Default)] - struct Build { - build: Option = "build", - host: Option> = "host", - target: Option> = "target", - build_dir: Option = "build-dir", - cargo: Option = "cargo", - rustc: Option = "rustc", - rustfmt: Option = "rustfmt", - cargo_clippy: Option = "cargo-clippy", - docs: Option = "docs", - compiler_docs: Option = "compiler-docs", - library_docs_private_items: Option = "library-docs-private-items", - docs_minification: Option = "docs-minification", - submodules: Option = "submodules", - gdb: Option = "gdb", - lldb: Option = "lldb", - nodejs: Option = "nodejs", - npm: Option = "npm", - python: Option = "python", - reuse: Option = "reuse", - locked_deps: Option = "locked-deps", - vendor: Option = "vendor", - full_bootstrap: Option = "full-bootstrap", - bootstrap_cache_path: Option = "bootstrap-cache-path", - extended: Option = "extended", - tools: Option> = "tools", - verbose: Option = "verbose", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - cargo_native_static: Option = "cargo-native-static", - low_priority: Option = "low-priority", - configure_args: Option> = "configure-args", - local_rebuild: Option = "local-rebuild", - print_step_timings: Option = "print-step-timings", - print_step_rusage: Option = "print-step-rusage", - check_stage: Option = "check-stage", - doc_stage: Option = "doc-stage", - build_stage: Option = "build-stage", - test_stage: Option = "test-stage", - install_stage: Option = "install-stage", - dist_stage: Option = "dist-stage", - bench_stage: Option = "bench-stage", - patch_binaries_for_nix: Option = "patch-binaries-for-nix", - // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally - metrics: Option = "metrics", - android_ndk: Option = "android-ndk", - optimized_compiler_builtins: Option = "optimized-compiler-builtins", - jobs: Option = "jobs", - compiletest_diff_tool: Option = "compiletest-diff-tool", - } -} - -define_config! { - /// TOML representation of various global install decisions. - struct Install { - prefix: Option = "prefix", - sysconfdir: Option = "sysconfdir", - docdir: Option = "docdir", - bindir: Option = "bindir", - libdir: Option = "libdir", - mandir: Option = "mandir", - datadir: Option = "datadir", - } -} - -define_config! { - /// TOML representation of how the LLVM build is configured. - struct Llvm { - optimize: Option = "optimize", - thin_lto: Option = "thin-lto", - release_debuginfo: Option = "release-debuginfo", - assertions: Option = "assertions", - tests: Option = "tests", - enzyme: Option = "enzyme", - plugins: Option = "plugins", - ccache: Option = "ccache", - static_libstdcpp: Option = "static-libstdcpp", - libzstd: Option = "libzstd", - ninja: Option = "ninja", - targets: Option = "targets", - experimental_targets: Option = "experimental-targets", - link_jobs: Option = "link-jobs", - link_shared: Option = "link-shared", - version_suffix: Option = "version-suffix", - clang_cl: Option = "clang-cl", - cflags: Option = "cflags", - cxxflags: Option = "cxxflags", - ldflags: Option = "ldflags", - use_libcxx: Option = "use-libcxx", - use_linker: Option = "use-linker", - allow_old_toolchain: Option = "allow-old-toolchain", - offload: Option = "offload", - polly: Option = "polly", - clang: Option = "clang", - enable_warnings: Option = "enable-warnings", - download_ci_llvm: Option = "download-ci-llvm", - build_config: Option> = "build-config", - enable_projects: Option = "enable-projects", - } -} - -define_config! { - struct Dist { - sign_folder: Option = "sign-folder", - upload_addr: Option = "upload-addr", - src_tarball: Option = "src-tarball", - compression_formats: Option> = "compression-formats", - compression_profile: Option = "compression-profile", - include_mingw_linker: Option = "include-mingw-linker", - vendor: Option = "vendor", - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum StringOrBool { - String(String), - Bool(bool), -} - -impl Default for StringOrBool { - fn default() -> StringOrBool { - StringOrBool::Bool(false) - } -} - -impl StringOrBool { - fn is_string_or_true(&self) -> bool { - matches!(self, Self::String(_) | Self::Bool(true)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RustOptimize { - String(String), - Int(u8), - Bool(bool), -} - -impl Default for RustOptimize { - fn default() -> RustOptimize { - RustOptimize::Bool(false) - } -} - -impl<'de> Deserialize<'de> for RustOptimize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(OptimizeVisitor) - } -} - -struct OptimizeVisitor; - -impl serde::de::Visitor<'_> for OptimizeVisitor { - type Value = RustOptimize; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - if matches!(value, "s" | "z") { - Ok(RustOptimize::String(value.to_string())) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - if matches!(value, 0..=3) { - Ok(RustOptimize::Int(value as u8)) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - Ok(RustOptimize::Bool(value)) - } -} - -fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { - format!( - r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# - ) -} - -impl RustOptimize { - pub(crate) fn is_release(&self) -> bool { - match &self { - RustOptimize::Bool(true) | RustOptimize::String(_) => true, - RustOptimize::Int(i) => *i > 0, - RustOptimize::Bool(false) => false, - } - } - - pub(crate) fn get_opt_level(&self) -> Option { - match &self { - RustOptimize::String(s) => Some(s.clone()), - RustOptimize::Int(i) => Some(i.to_string()), - RustOptimize::Bool(_) => None, - } - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum StringOrInt { - String(String), - Int(i64), -} - -impl<'de> Deserialize<'de> for LldMode { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct LldModeVisitor; - - impl serde::de::Visitor<'_> for LldModeVisitor { - type Value = LldMode; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("one of true, 'self-contained' or 'external'") - } - - fn visit_bool(self, v: bool) -> Result - where - E: serde::de::Error, - { - Ok(if v { LldMode::External } else { LldMode::Unused }) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - match v { - "external" => Ok(LldMode::External), - "self-contained" => Ok(LldMode::SelfContained), - _ => Err(E::custom("unknown mode {v}")), - } - } - } - - deserializer.deserialize_any(LldModeVisitor) - } -} - -define_config! { - /// TOML representation of how the Rust build is configured. - struct Rust { - optimize: Option = "optimize", - debug: Option = "debug", - codegen_units: Option = "codegen-units", - codegen_units_std: Option = "codegen-units-std", - rustc_debug_assertions: Option = "debug-assertions", - randomize_layout: Option = "randomize-layout", - std_debug_assertions: Option = "debug-assertions-std", - overflow_checks: Option = "overflow-checks", - overflow_checks_std: Option = "overflow-checks-std", - debug_logging: Option = "debug-logging", - debuginfo_level: Option = "debuginfo-level", - debuginfo_level_rustc: Option = "debuginfo-level-rustc", - debuginfo_level_std: Option = "debuginfo-level-std", - debuginfo_level_tools: Option = "debuginfo-level-tools", - debuginfo_level_tests: Option = "debuginfo-level-tests", - backtrace: Option = "backtrace", - incremental: Option = "incremental", - parallel_compiler: Option = "parallel-compiler", - default_linker: Option = "default-linker", - channel: Option = "channel", - description: Option = "description", - musl_root: Option = "musl-root", - rpath: Option = "rpath", - strip: Option = "strip", - frame_pointers: Option = "frame-pointers", - stack_protector: Option = "stack-protector", - verbose_tests: Option = "verbose-tests", - optimize_tests: Option = "optimize-tests", - codegen_tests: Option = "codegen-tests", - omit_git_hash: Option = "omit-git-hash", - dist_src: Option = "dist-src", - save_toolstates: Option = "save-toolstates", - codegen_backends: Option> = "codegen-backends", - llvm_bitcode_linker: Option = "llvm-bitcode-linker", - lld: Option = "lld", - lld_mode: Option = "use-lld", - llvm_tools: Option = "llvm-tools", - deny_warnings: Option = "deny-warnings", - backtrace_on_ice: Option = "backtrace-on-ice", - verify_llvm_ir: Option = "verify-llvm-ir", - thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", - remap_debuginfo: Option = "remap-debuginfo", - jemalloc: Option = "jemalloc", - test_compare_mode: Option = "test-compare-mode", - llvm_libunwind: Option = "llvm-libunwind", - control_flow_guard: Option = "control-flow-guard", - ehcont_guard: Option = "ehcont-guard", - new_symbol_mangling: Option = "new-symbol-mangling", - profile_generate: Option = "profile-generate", - profile_use: Option = "profile-use", - // ignored; this is set from an env var set by bootstrap.py - download_rustc: Option = "download-rustc", - lto: Option = "lto", - validate_mir_opts: Option = "validate-mir-opts", - std_features: Option> = "std-features", - } -} - -define_config! { - /// TOML representation of how each build target is configured. - struct TomlTarget { - cc: Option = "cc", - cxx: Option = "cxx", - ar: Option = "ar", - ranlib: Option = "ranlib", - default_linker: Option = "default-linker", - linker: Option = "linker", - split_debuginfo: Option = "split-debuginfo", - llvm_config: Option = "llvm-config", - llvm_has_rust_patches: Option = "llvm-has-rust-patches", - llvm_filecheck: Option = "llvm-filecheck", - llvm_libunwind: Option = "llvm-libunwind", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - rpath: Option = "rpath", - crt_static: Option = "crt-static", - musl_root: Option = "musl-root", - musl_libdir: Option = "musl-libdir", - wasi_root: Option = "wasi-root", - qemu_rootfs: Option = "qemu-rootfs", - no_std: Option = "no-std", - codegen_backends: Option> = "codegen-backends", - runner: Option = "runner", - } -} - +use crate::prelude::*; +use std::path::absolute; +use crate::core::config::config_part2::{set, threads_from_config}; impl Config { - pub fn default_opts() -> Config { - Config { - bypass_bootstrap_lock: false, - llvm_optimize: true, - ninja_in_file: true, - llvm_static_stdcpp: false, - llvm_libzstd: false, - backtrace: true, - rust_optimize: RustOptimize::Bool(true), - rust_optimize_tests: true, - rust_randomize_layout: false, - submodules: None, - docs: true, - docs_minification: true, - rust_rpath: true, - rust_strip: false, - channel: "dev".to_string(), - codegen_tests: true, - rust_dist_src: true, - rust_codegen_backends: vec!["llvm".to_owned()], - deny_warnings: true, - bindir: "bin".into(), - dist_include_mingw_linker: true, - dist_compression_profile: "fast".into(), - - stdout_is_tty: std::io::stdout().is_terminal(), - stderr_is_tty: std::io::stderr().is_terminal(), - - // set by build.rs - build: TargetSelection::from_user(env!("BUILD_TRIPLE")), - - src: { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - manifest_dir.parent().unwrap().parent().unwrap().to_owned() - }, - out: PathBuf::from("build"), - - // This is needed by codegen_ssa on macOS to ship `llvm-objcopy` aliased to - // `rust-objcopy` to workaround bad `strip`s on macOS. - llvm_tools_enabled: true, - - ..Default::default() - } - } - pub(crate) fn get_builder_toml(&self, build_name: &str) -> Result { - if self.dry_run() { - return Ok(TomlConfig::default()); - } - - let builder_config_path = - self.out.join(self.build.triple).join(build_name).join(BUILDER_CONFIG_FILENAME); - Self::get_toml(&builder_config_path) - } - #[cfg(test)] - pub(crate) fn get_toml(_: &Path) -> Result { - Ok(TomlConfig::default()) - } - - #[cfg(not(test))] - pub(crate) fn get_toml(file: &Path) -> Result { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .inspect_err(|_| { - if let Ok(Some(changes)) = toml::from_str(&contents) - .and_then(|table: toml::Value| ChangeIdWrapper::deserialize(table)) - .map(|change_id| change_id.inner.map(crate::find_recent_config_change_ids)) - { - if !changes.is_empty() { - println!( - "WARNING: There have been changes to x.py since you last updated:\n{}", - crate::human_readable_changes(&changes) - ); - } - } - }) - } - pub fn parse(flags: Flags) -> Config { - Self::parse_inner(flags, Self::get_toml) - } pub(crate) fn parse_inner( mut flags: Flags, @@ -1335,53 +34,15 @@ impl Config { // Infer the rest of the configuration. - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely different machine from where it was compiled. - let mut cmd = helpers::git(None); - // NOTE: we cannot support running from outside the repository because the only other path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded rather than compiled locally. - // We still support running outside the repository if we find we aren't in a git directory. - - // NOTE: We get a relative path from git to work around an issue on MSYS/mingw. If we used an absolute path, - // and end up using MSYS's git rather than git-for-windows, we would get a unix-y MSYS path. But as bootstrap - // has already been (kinda-cross-)compiled to Windows land, we require a normal Windows path. - cmd.arg("rev-parse").arg("--show-cdup"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .as_command_mut() - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root_relative = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes, - // and to resolve any relative components. - let git_root = env::current_dir() - .unwrap() - .join(PathBuf::from(git_root_relative.trim())) - .canonicalize() - .unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let git_root = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => git_root, - }; - // If this doesn't have at least `stage0`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if git_root.join("src").join("stage0").exists() { - config.src = git_root; - } + config.src = if let Some(src) = flags.src { + src + } else if let Some(src) = build_src_from_toml { + src } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + manifest_dir.parent().unwrap().parent().unwrap().to_owned() + }; if cfg!(test) { // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. @@ -1393,12 +54,15 @@ impl Config { .to_path_buf(); } - config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file(); + config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file( + &toml.stage0_path.expect("stage0_path must be set"), + ); // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. let toml_path = flags .config .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_GENERATED_CONFIG").map(PathBuf::from)) .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); let using_default_path = toml_path.is_none(); let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); @@ -1406,7 +70,7 @@ impl Config { toml_path = config.src.join(toml_path); } - let file_content = t!(fs::read_to_string(config.src.join("src/ci/channel"))); + let file_content = t!(fs::read_to_string(&config.ci.channel_file)); let ci_channel = file_content.trim_end(); // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, @@ -1417,7 +81,7 @@ impl Config { } else { toml_path.clone() }); - get_toml(&toml_path).unwrap_or_else(|e| { + get_toml::get_toml(&toml_path).unwrap_or_else(|e| { eprintln!("ERROR: Failed to parse '{}': {e}", toml_path.display()); exit!(2); }) @@ -1463,7 +127,7 @@ impl Config { let mut override_toml = TomlConfig::default(); for option in flags.set.iter() { - fn get_table(option: &str) -> Result { +pub fn get_table(option: &str) -> Result { toml::from_str(option).and_then(|table: toml::Value| TomlConfig::deserialize(table)) } @@ -1492,8 +156,35 @@ impl Config { } toml.merge(override_toml, ReplaceOpt::Override); + let build_src = toml.build.as_ref().and_then(|b| b.src.clone()); + + let Ci { + channel_file, + version_file, + tools_dir, + llvm_project_dir, + gcc_dir, + } = toml.ci.unwrap_or_default(); + + set(&mut config.ci.channel_file, channel_file.map(PathBuf::from)); + set(&mut config.ci.version_file, version_file.map(PathBuf::from)); + set(&mut config.ci.tools_dir, tools_dir.map(PathBuf::from)); + set(&mut config.ci.llvm_project_dir, llvm_project_dir.map(PathBuf::from)); + set(&mut config.ci.gcc_dir, gcc_dir.map(PathBuf::from)); + config.change_id = toml.change_id.inner; + if let Some(nix) = toml.nix { + config.nixpkgs_path = nix.nixpkgs_path; + config.rust_overlay_path = nix.rust_overlay_path; + config.rust_bootstrap_nix_path = nix.rust_bootstrap_nix_path; + config.configuration_nix_path = nix.configuration_nix_path; + config.rust_src_flake_path = nix.rust_src_flake_path; + } + + // Resolve Nix paths dynamically if not already set + config.resolve_nix_paths().expect("Failed to resolve Nix paths"); + let Build { build, host, @@ -1543,6 +234,7 @@ impl Config { optimized_compiler_builtins, jobs, compiletest_diff_tool, + src: build_src_from_toml, } = toml.build.unwrap_or_default(); config.jobs = Some(threads_from_config(flags.jobs.unwrap_or(jobs.unwrap_or(0)))); @@ -1598,7 +290,7 @@ impl Config { }; // NOTE: it's important this comes *after* we set `initial_rustc` just above. - if config.dry_run() { + if config.dry_run { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); config.out = dir; @@ -1661,7 +353,12 @@ impl Config { config.sysconfdir = sysconfdir.map(PathBuf::from); config.datadir = datadir.map(PathBuf::from); config.docdir = docdir.map(PathBuf::from); - set(&mut config.bindir, bindir.map(PathBuf::from)); + // Handle bindir specifically, as it's not an Option in Config + if let Some(b) = bindir { + config.bindir = PathBuf::from(b); + } else if let Some(p) = &config.prefix { + config.bindir = p.join("bin"); + } config.libdir = libdir.map(PathBuf::from); config.mandir = mandir.map(PathBuf::from); } @@ -1701,19 +398,19 @@ impl Config { let default = config.channel == "dev"; config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(default); - config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - config.cargo_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/cargo")); + config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); // config.src is still the overall source root + config.cargo_info = GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("cargo")); config.rust_analyzer_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer")); + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("rust-analyzer")); config.clippy_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/clippy")); - config.miri_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/miri")); + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("clippy")); + config.miri_info = GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("miri")); config.rustfmt_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rustfmt")); + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("rustfmt")); config.enzyme_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/enzyme")); - config.in_tree_llvm_info = GitInfo::new(false, &config.src.join("src/llvm-project")); - config.in_tree_gcc_info = GitInfo::new(false, &config.src.join("src/gcc")); + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("enzyme")); + config.in_tree_llvm_info = GitInfo::new(false, &config.ci.llvm_project_dir); + config.in_tree_gcc_info = GitInfo::new(false, &config.ci.gcc_dir); if let Some(rust) = toml.rust { let Rust { @@ -1882,7 +579,7 @@ impl Config { ); let channel = config - .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) + .read_file_by_commit(&config.ci.channel_file, commit) .trim() .to_owned(); @@ -2267,24 +964,10 @@ impl Config { config } - pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } /// Runs a command, printing out nice contextual information if it fails. /// Exits if the command failed to execute at all, otherwise returns its /// `status.success()`. - #[deprecated = "use `Builder::try_run` instead where possible"] - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(|| println!("running: {cmd:?}")); - build_helper::util::try_run(cmd, self.is_verbose()) - } pub(crate) fn test_args(&self) -> Vec<&str> { let mut test_args = match self.cmd { @@ -2331,11 +1014,11 @@ impl Config { .trim() .to_owned(); let version = - self.read_file_by_commit(&PathBuf::from("src/version"), commit).trim().to_owned(); + self.read_file_by_commit(&self.ci.version_file, commit).trim().to_owned(); (channel, version) } else { - let channel = fs::read_to_string(self.src.join("src/ci/channel")); - let version = fs::read_to_string(self.src.join("src/version")); + let channel = fs::read_to_string(&self.ci.channel_file); + let version = fs::read_to_string(&self.ci.version_file); match (channel, version) { (Ok(channel), Ok(version)) => { (channel.trim().to_owned(), version.trim().to_owned()) @@ -2408,7 +1091,7 @@ impl Config { /// This is computed on demand since LLVM might have to first be downloaded from CI. pub(crate) fn llvm_link_shared(&self) -> bool { let mut opt = self.llvm_link_shared.get(); - if opt.is_none() && self.dry_run() { + if opt.is_none() && self.dry_run { // just assume static for now - dynamic linking isn't supported on all platforms return false; } @@ -2439,7 +1122,7 @@ impl Config { pub(crate) fn download_rustc_commit(&self) -> Option<&str> { static DOWNLOAD_RUSTC: OnceLock> = OnceLock::new(); - if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { + if self.dry_run && DOWNLOAD_RUSTC.get().is_none() { // avoid trying to actually download the commit return self.download_rustc_commit.as_deref(); } @@ -2465,7 +1148,7 @@ impl Config { } if let Some(config_path) = &self.config { - let ci_config_toml = match self.get_builder_toml("ci-rustc") { + let ci_config_toml = match get_builder_toml::get_builder_toml(self, "ci-rustc") { Ok(ci_config_toml) => ci_config_toml, Err(e) if e.to_string().contains("unknown field") => { println!("WARNING: CI rustc has some fields that are no longer supported in bootstrap; download-rustc will be disabled."); @@ -2478,7 +1161,7 @@ impl Config { }, }; - let current_config_toml = Self::get_toml(config_path).unwrap(); + let current_config_toml = get_toml::get_toml(config_path).unwrap(); // Check the config compatibility // FIXME: this doesn't cover `--set` flags yet. @@ -2511,7 +1194,7 @@ impl Config { RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), RustfmtState::Unavailable => None, r @ RustfmtState::LazyEvaluated => { - if self.dry_run() { + if self.dry_run { return Some(PathBuf::new()); } let path = self.maybe_download_rustfmt(); @@ -2745,7 +1428,7 @@ impl Config { pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { use build_helper::util::fail; - if self.dry_run() { + if self.dry_run { return; } @@ -2780,7 +1463,7 @@ impl Config { } /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit( +pub fn download_ci_rustc_commit( &self, download_rustc: Option, llvm_assertions: bool, @@ -2858,7 +1541,7 @@ impl Config { Some(commit) } - fn parse_download_ci_llvm( +pub fn parse_download_ci_llvm( &self, download_ci_llvm: Option, asserts: bool, @@ -2950,257 +1633,3 @@ impl Config { Some(commit.to_string()) } } - -/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. -/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. -#[cfg(not(feature = "bootstrap-self-test"))] -pub(crate) fn check_incompatible_options_for_ci_llvm( - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; - } - - macro_rules! warn { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let (Some(current_llvm_config), Some(ci_llvm_config)) = - (current_config_toml.llvm, ci_config_toml.llvm) - else { - return Ok(()); - }; - - let Llvm { - optimize, - thin_lto, - release_debuginfo, - assertions: _, - tests: _, - plugins, - ccache: _, - static_libstdcpp: _, - libzstd, - ninja: _, - targets, - experimental_targets, - link_jobs: _, - link_shared: _, - version_suffix, - clang_cl, - cflags, - cxxflags, - ldflags, - use_libcxx, - use_linker, - allow_old_toolchain, - offload, - polly, - clang, - enable_warnings, - download_ci_llvm: _, - build_config, - enzyme, - enable_projects: _, - } = ci_llvm_config; - - err!(current_llvm_config.optimize, optimize); - err!(current_llvm_config.thin_lto, thin_lto); - err!(current_llvm_config.release_debuginfo, release_debuginfo); - err!(current_llvm_config.libzstd, libzstd); - err!(current_llvm_config.targets, targets); - err!(current_llvm_config.experimental_targets, experimental_targets); - err!(current_llvm_config.clang_cl, clang_cl); - err!(current_llvm_config.version_suffix, version_suffix); - err!(current_llvm_config.cflags, cflags); - err!(current_llvm_config.cxxflags, cxxflags); - err!(current_llvm_config.ldflags, ldflags); - err!(current_llvm_config.use_libcxx, use_libcxx); - err!(current_llvm_config.use_linker, use_linker); - err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); - err!(current_llvm_config.offload, offload); - err!(current_llvm_config.polly, polly); - err!(current_llvm_config.clang, clang); - err!(current_llvm_config.build_config, build_config); - err!(current_llvm_config.plugins, plugins); - err!(current_llvm_config.enzyme, enzyme); - - warn!(current_llvm_config.enable_warnings, enable_warnings); - - Ok(()) -} - -/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. -/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. -fn check_incompatible_options_for_ci_rustc( - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `rust.{}` is incompatible with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; - } - - macro_rules! warn { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `rust.{}` has no effect with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let (Some(current_rust_config), Some(ci_rust_config)) = - (current_config_toml.rust, ci_config_toml.rust) - else { - return Ok(()); - }; - - let Rust { - // Following options are the CI rustc incompatible ones. - optimize, - randomize_layout, - debug_logging, - debuginfo_level_rustc, - llvm_tools, - llvm_bitcode_linker, - lto, - stack_protector, - strip, - lld_mode, - jemalloc, - rpath, - channel, - description, - incremental, - default_linker, - std_features, - - // Rest of the options can simply be ignored. - debug: _, - codegen_units: _, - codegen_units_std: _, - rustc_debug_assertions: _, - std_debug_assertions: _, - overflow_checks: _, - overflow_checks_std: _, - debuginfo_level: _, - debuginfo_level_std: _, - debuginfo_level_tools: _, - debuginfo_level_tests: _, - backtrace: _, - parallel_compiler: _, - musl_root: _, - verbose_tests: _, - optimize_tests: _, - codegen_tests: _, - omit_git_hash: _, - dist_src: _, - save_toolstates: _, - codegen_backends: _, - lld: _, - deny_warnings: _, - backtrace_on_ice: _, - verify_llvm_ir: _, - thin_lto_import_instr_limit: _, - remap_debuginfo: _, - test_compare_mode: _, - llvm_libunwind: _, - control_flow_guard: _, - ehcont_guard: _, - new_symbol_mangling: _, - profile_generate: _, - profile_use: _, - download_rustc: _, - validate_mir_opts: _, - frame_pointers: _, - } = ci_rust_config; - - // There are two kinds of checks for CI rustc incompatible options: - // 1. Checking an option that may change the compiler behaviour/output. - // 2. Checking an option that have no effect on the compiler behaviour/output. - // - // If the option belongs to the first category, we call `err` macro for a hard error; - // otherwise, we just print a warning with `warn` macro. - - err!(current_rust_config.optimize, optimize); - err!(current_rust_config.randomize_layout, randomize_layout); - err!(current_rust_config.debug_logging, debug_logging); - err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc); - err!(current_rust_config.rpath, rpath); - err!(current_rust_config.strip, strip); - err!(current_rust_config.lld_mode, lld_mode); - err!(current_rust_config.llvm_tools, llvm_tools); - err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker); - err!(current_rust_config.jemalloc, jemalloc); - err!(current_rust_config.default_linker, default_linker); - err!(current_rust_config.stack_protector, stack_protector); - err!(current_rust_config.lto, lto); - err!(current_rust_config.std_features, std_features); - - warn!(current_rust_config.channel, channel); - warn!(current_rust_config.description, description); - warn!(current_rust_config.incremental, incremental); - - Ok(()) -} - -fn set(field: &mut T, val: Option) { - if let Some(v) = val { - *field = v; - } -} - -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, - } -} diff --git a/standalonex/src/bootstrap/src/core/config/config_part6.rs b/standalonex/src/bootstrap/src/core/config/config_part6.rs new file mode 100644 index 00000000..038a7046 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/config_part6.rs @@ -0,0 +1,63 @@ +pub struct OptimizeVisitor; + +impl serde::de::Visitor<'_> for OptimizeVisitor { + type Value = RustOptimize; + +fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + +fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + if matches!(value, "s" | "z") { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + +fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + +fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) + } +} + +pub fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + +impl RustOptimize { + pub(crate) fn is_release(&self) -> bool { + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } + } + + pub(crate) fn get_opt_level(&self) -> Option { + match &self { + RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), + RustOptimize::Bool(_) => None, + } + } +} + diff --git a/standalonex/src/bootstrap/src/core/config/config_part7.rs b/standalonex/src/bootstrap/src/core/config/config_part7.rs new file mode 100644 index 00000000..fd40910d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/config_part7.rs @@ -0,0 +1,4 @@ + + + + diff --git a/standalonex/src/bootstrap/src/core/config/config_toml.rs b/standalonex/src/bootstrap/src/core/config/config_toml.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config/config_types.rs b/standalonex/src/bootstrap/src/core/config/config_types.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config/config_utils.rs b/standalonex/src/bootstrap/src/core/config/config_utils.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config/debug_info_level.rs b/standalonex/src/bootstrap/src/core/config/debug_info_level.rs new file mode 100644 index 00000000..4433c3d6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/debug_info_level.rs @@ -0,0 +1,61 @@ +use crate::prelude::*; +use serde::Deserializer; +use crate::core::config::string_or_int::StringOrInt; + +#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] +pub enum DebuginfoLevel { + #[default] + None, + LineDirectivesOnly, + LineTablesOnly, + Limited, + Full, +} + +// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only +// deserializes i64, and derive() only generates visit_u64 +impl<'de> Deserialize<'de> for DebuginfoLevel { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + + + Ok(match Deserialize::deserialize(deserializer)? { + StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, + StringOrInt::Int(0) => DebuginfoLevel::None, + StringOrInt::String(s) if s == "line-directives-only" => { + DebuginfoLevel::LineDirectivesOnly + } + StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, + StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, + StringOrInt::Int(1) => DebuginfoLevel::Limited, + StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, + StringOrInt::Int(2) => DebuginfoLevel::Full, + StringOrInt::Int(n) => { + let other = serde::de::Unexpected::Signed(n); + return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); + } + StringOrInt::String(s) => { + let other = serde::de::Unexpected::Str(&s); + return Err(D::Error::invalid_value( + other, + &"expected none, line-tables-only, limited, or full", + )); + } + }) + } +} +/// Suitable for passing to `-C debuginfo` +impl Display for DebuginfoLevel { +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DebuginfoLevel::*; + f.write_str(match self { + None => "0", + LineDirectivesOnly => "line-directives-only", + LineTablesOnly => "line-tables-only", + Limited => "1", + Full => "2", + }) + } +} diff --git a/standalonex/src/bootstrap/src/core/config/dist.rs b/standalonex/src/bootstrap/src/core/config/dist.rs new file mode 100644 index 00000000..85c375af --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/dist.rs @@ -0,0 +1,13 @@ +use config_macros::define_config; + +define_config! { + struct Dist { + sign_folder: Option = "sign-folder", + upload_addr: Option = "upload-addr", + src_tarball: Option = "src-tarball", + compression_formats: Option> = "compression-formats", + compression_profile: Option = "compression-profile", + include_mingw_linker: Option = "include-mingw-linker", + vendor: Option = "vendor", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/dry_run.rs b/standalonex/src/bootstrap/src/core/config/dry_run.rs new file mode 100644 index 00000000..75c550d3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/dry_run.rs @@ -0,0 +1,18 @@ +/// This file is embedded in the overlay directory of the tarball sources. It is +/// useful in scenarios where developers want to see how the tarball sources were +/// generated. +/// +/// We also use this file to compare the host's config.toml against the CI rustc builder +/// configuration to detect any incompatible options. +pub const BUILDER_CONFIG_FILENAME: &str = "builder-config"; + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} diff --git a/standalonex/src/bootstrap/src/core/config/flags.rs b/standalonex/src/bootstrap/src/core/config/flags.rs index bfeb8115..1d838f92 100644 --- a/standalonex/src/bootstrap/src/core/config/flags.rs +++ b/standalonex/src/bootstrap/src/core/config/flags.rs @@ -11,25 +11,24 @@ use crate::core::build_steps::setup::Profile; use crate::core::builder::{Builder, Kind}; use crate::core::config::{Config, TargetSelectionList, target_selection_list}; use crate::{Build, DocTests}; +pub use crate::core::config::subcommand::get_completion; -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Color { - Always, - Never, - #[default] - Auto, -} -/// Whether to deny warnings, emit them as warnings, or use the default behavior -#[derive(Copy, Clone, Default, Debug, ValueEnum)] + +#[derive(Clone, ValueEnum)] pub enum Warnings { + Default, Deny, Warn, - #[default] - Default, } -/// Deserialized version of all flags for this compile. +#[derive(Clone, ValueEnum)] +pub enum Color { + Auto, + Always, + Never, +} + #[derive(Debug, Parser)] #[command( override_usage = "x.py [options] [...]", @@ -88,7 +87,7 @@ pub struct Flags { /// Indicates whether to dump the work done from bootstrap shims #[arg(global = true, long)] pub dump_bootstrap_shims: bool, - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] + #[arg(global = true, value_hint = clap::ValueHint::Other, value_name = "N")] /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) pub stage: Option, @@ -216,426 +215,9 @@ impl Flags { } } -fn normalize_args(args: &[String]) -> Vec { +pub fn normalize_args(args: &[String]) -> Vec { let first = String::from("x.py"); let it = std::iter::once(first).chain(args.iter().cloned()); it.collect() } -#[derive(Debug, Clone, Default, clap::Subcommand)] -pub enum Subcommand { - #[command(aliases = ["b"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example, for a quick build of a usable - compiler: - ./x.py build --stage 1 library/std - This will build a compiler and standard library from the local source code. - Once this is done, build/$ARCH/stage1 contains a usable compiler. - If no arguments are passed then the default artifacts for that stage are - compiled. For example: - ./x.py build --stage 0 - ./x.py build ")] - /// Compile either the compiler or libraries - #[default] - Build, - #[command(aliases = ["c"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example: - ./x.py check library/std - If no arguments are passed then many artifacts are checked.")] - /// Compile either the compiler or libraries, using cargo check - Check { - #[arg(long)] - /// Check all targets - all_targets: bool, - }, - /// Run Clippy (uses rustup/cargo-installed clippy binary) - #[command(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run clippy against. For example: - ./x.py clippy library/core - ./x.py clippy library/core library/proc_macro")] - Clippy { - #[arg(long)] - fix: bool, - #[arg(long, requires = "fix")] - allow_dirty: bool, - #[arg(long, requires = "fix")] - allow_staged: bool, - /// clippy lints to allow - #[arg(global = true, short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] - allow: Vec, - /// clippy lints to deny - #[arg(global = true, short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] - deny: Vec, - /// clippy lints to warn on - #[arg(global = true, short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] - warn: Vec, - /// clippy lints to forbid - #[arg(global = true, short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] - forbid: Vec, - }, - /// Run cargo fix - #[command(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run `cargo fix` against. For example: - ./x.py fix library/core - ./x.py fix library/core library/proc_macro")] - Fix, - #[command( - name = "fmt", - long_about = "\n - Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if - formatting is correct and fails if it is not. For example: - ./x.py fmt - ./x.py fmt --check" - )] - /// Run rustfmt - Format { - /// check formatting instead of applying - #[arg(long)] - check: bool, - - /// apply to all appropriate files, not just those that have been modified - #[arg(long)] - all: bool, - }, - #[command(aliases = ["d"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories of documentation - to build. For example: - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc src/doc/book library/std - ./x.py doc library/std --json - ./x.py doc library/std --open - If no arguments are passed then everything is documented: - ./x.py doc - ./x.py doc --stage 1")] - /// Build documentation - Doc { - #[arg(long)] - /// open the docs in a browser - open: bool, - #[arg(long)] - /// render the documentation in JSON format in addition to the usual HTML format - json: bool, - }, - #[command(aliases = ["t"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to test directories that - should be compiled and run. For example: - ./x.py test tests/ui - ./x.py test library/std --test-args hash_map - ./x.py test library/std --stage 0 --no-doc - ./x.py test tests/ui --bless - ./x.py test tests/ui --compare-mode next-solver - Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; - just like `build library/std --stage N` it tests the compiler produced by the previous - stage. - Execute tool tests with a tool name argument: - ./x.py test tidy - If no arguments are passed then the complete artifacts for that stage are - compiled and tested. - ./x.py test - ./x.py test --stage 1")] - /// Build and run some test suites - Test { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - /// extra options to pass the compiler when running compiletest tests - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - compiletest_rustc_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - #[arg(long)] - /// whether to automatically update stderr/stdout files - bless: bool, - #[arg(long)] - /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) - extra_checks: Option, - #[arg(long)] - /// rerun tests even if the inputs are unchanged - force_rerun: bool, - #[arg(long)] - /// only run tests that result has been changed - only_modified: bool, - #[arg(long, value_name = "COMPARE MODE")] - /// mode describing what file the actual ui output will be compared to - compare_mode: Option, - #[arg(long, value_name = "check | build | run")] - /// force {check,build,run}-pass tests to this mode. - pass: Option, - #[arg(long, value_name = "auto | always | never")] - /// whether to execute run-* tests - run: Option, - #[arg(long)] - /// enable this to generate a Rustfix coverage file, which is saved in - /// `//rustfix_missing_coverage.txt` - rustfix_coverage: bool, - }, - /// Build and run some test suites *in Miri* - Miri { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - }, - /// Build and run some benchmarks - Bench { - #[arg(long, allow_hyphen_values(true))] - test_args: Vec, - }, - /// Clean out build directories - Clean { - #[arg(long)] - /// Clean the entire build directory (not used by default) - all: bool, - #[arg(long, value_name = "N")] - /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. - stage: Option, - }, - /// Build distribution artifacts - Dist, - /// Install distribution artifacts - Install, - #[command(aliases = ["r"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to tools to build and run. For - example: - ./x.py run src/tools/bump-stage0 - At least a tool needs to be called.")] - /// Run tools contained in this repository - Run { - /// arguments for the tool - #[arg(long, allow_hyphen_values(true))] - args: Vec, - }, - /// Set up the environment for development - #[command(long_about = format!( - "\n -x.py setup creates a `config.toml` which changes the defaults for x.py itself, -as well as setting up a git pre-push hook, VS Code config and toolchain link. -Arguments: - This subcommand accepts a 'profile' to use for builds. For example: - ./x.py setup library - The profile is optional and you will be prompted interactively if it is not given. - The following profiles are available: -{} - To only set up the git hook, editor config or toolchain link, you may use - ./x.py setup hook - ./x.py setup editor - ./x.py setup link", Profile::all_for_help(" ").trim_end()))] - Setup { - /// Either the profile for `config.toml` or another setup action. - /// May be omitted to set up interactively - #[arg(value_name = "|hook|editor|link")] - profile: Option, - }, - /// Suggest a subset of tests to run, based on modified files - #[command(long_about = "\n")] - Suggest { - /// run suggested tests - #[arg(long)] - run: bool, - }, - /// Vendor dependencies - Vendor { - /// Additional `Cargo.toml` to sync and vendor - #[arg(long)] - sync: Vec, - /// Always include version in subdir name - #[arg(long)] - versioned_dirs: bool, - }, - /// Perform profiling and benchmarking of the compiler using the - /// `rustc-perf-wrapper` tool. - /// - /// You need to pass arguments after `--`, e.g.`x perf -- cachegrind`. - Perf {}, -} - -impl Subcommand { - pub fn kind(&self) -> Kind { - match self { - Subcommand::Bench { .. } => Kind::Bench, - Subcommand::Build { .. } => Kind::Build, - Subcommand::Check { .. } => Kind::Check, - Subcommand::Clippy { .. } => Kind::Clippy, - Subcommand::Doc { .. } => Kind::Doc, - Subcommand::Fix { .. } => Kind::Fix, - Subcommand::Format { .. } => Kind::Format, - Subcommand::Test { .. } => Kind::Test, - Subcommand::Miri { .. } => Kind::Miri, - Subcommand::Clean { .. } => Kind::Clean, - Subcommand::Dist { .. } => Kind::Dist, - Subcommand::Install { .. } => Kind::Install, - Subcommand::Run { .. } => Kind::Run, - Subcommand::Setup { .. } => Kind::Setup, - Subcommand::Suggest { .. } => Kind::Suggest, - Subcommand::Vendor { .. } => Kind::Vendor, - Subcommand::Perf { .. } => Kind::Perf, - } - } - - pub fn compiletest_rustc_args(&self) -> Vec<&str> { - match *self { - Subcommand::Test { ref compiletest_rustc_args, .. } => { - compiletest_rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - } - } - - pub fn fail_fast(&self) -> bool { - match *self { - Subcommand::Test { no_fail_fast, .. } | Subcommand::Miri { no_fail_fast, .. } => { - !no_fail_fast - } - _ => false, - } - } - - pub fn doc_tests(&self) -> DocTests { - match *self { - Subcommand::Test { doc, no_doc, .. } | Subcommand::Miri { no_doc, doc, .. } => { - if doc { - DocTests::Only - } else if no_doc { - DocTests::No - } else { - DocTests::Yes - } - } - _ => DocTests::Yes, - } - } - - pub fn bless(&self) -> bool { - match *self { - Subcommand::Test { bless, .. } => bless, - _ => false, - } - } - - pub fn extra_checks(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), - _ => None, - } - } - - pub fn only_modified(&self) -> bool { - match *self { - Subcommand::Test { only_modified, .. } => only_modified, - _ => false, - } - } - - pub fn force_rerun(&self) -> bool { - match *self { - Subcommand::Test { force_rerun, .. } => force_rerun, - _ => false, - } - } - - pub fn rustfix_coverage(&self) -> bool { - match *self { - Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, - _ => false, - } - } - - pub fn compare_mode(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn pass(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn run(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn open(&self) -> bool { - match *self { - Subcommand::Doc { open, .. } => open, - _ => false, - } - } - - pub fn json(&self) -> bool { - match *self { - Subcommand::Doc { json, .. } => json, - _ => false, - } - } - - pub fn vendor_versioned_dirs(&self) -> bool { - match *self { - Subcommand::Vendor { versioned_dirs, .. } => versioned_dirs, - _ => false, - } - } - - pub fn vendor_sync_args(&self) -> Vec { - match self { - Subcommand::Vendor { sync, .. } => sync.clone(), - _ => vec![], - } - } -} - -/// Returns the shell completion for a given shell, if the result differs from the current -/// content of `path`. If `path` does not exist, always returns `Some`. -pub fn get_completion(shell: G, path: &Path) -> Option { - let mut cmd = Flags::command(); - let current = if !path.exists() { - String::new() - } else { - std::fs::read_to_string(path).unwrap_or_else(|_| { - eprintln!("couldn't read {}", path.display()); - crate::exit!(1) - }) - }; - let mut buf = Vec::new(); - clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); - if buf == current.as_bytes() { - return None; - } - Some(String::from_utf8(buf).expect("completion script should be UTF-8")) -} diff --git a/standalonex/src/bootstrap/src/core/config/install.rs b/standalonex/src/bootstrap/src/core/config/install.rs new file mode 100644 index 00000000..14c9cbd7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/install.rs @@ -0,0 +1,14 @@ +use config_macros::define_config; + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option = "prefix", + sysconfdir: Option = "sysconfdir", + docdir: Option = "docdir", + bindir: Option = "bindir", + libdir: Option = "libdir", + mandir: Option = "mandir", + datadir: Option = "datadir", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/lld_mode.rs b/standalonex/src/bootstrap/src/core/config/lld_mode.rs new file mode 100644 index 00000000..13f7aada --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/lld_mode.rs @@ -0,0 +1,76 @@ +use crate::prelude::*; +use serde::Deserializer; + +/// LLD in bootstrap works like this: +/// - Self-contained lld: use `rust-lld` from the compiler's sysroot +/// - External: use an external `lld` binary +/// +/// It is configured depending on the target: +/// 1) Everything except MSVC +/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` +/// - External: `-Clinker-flavor=gnu-lld-cc` +/// 2) MSVC +/// - Self-contained: `-Clinker=` +/// - External: `-Clinker=lld` +use crate::prelude::*; +use serde::de::Error; + +#[derive(Copy, Clone, Default, Debug, PartialEq)] +pub enum LldMode { + /// Do not use LLD + #[default] + Unused, + /// Use `rust-lld` from the compiler's sysroot + SelfContained, + /// Use an externally provided `lld` binary. + /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has + /// to be in $PATH. + External, +} + +impl LldMode { + pub fn is_used(&self) -> bool { + match self { + LldMode::SelfContained | LldMode::External => true, + LldMode::Unused => false, + } + } +} + + +impl<'de> Deserialize<'de> for LldMode { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct LldModeVisitor; + + impl serde::de::Visitor<'_> for LldModeVisitor { + type Value = LldMode; + +fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("one of true, 'self-contained' or 'external'") + } + +fn visit_bool(self, v: bool) -> Result + where + E: serde::de::Error, + { + Ok(if v { LldMode::External } else { LldMode::Unused }) + } + +fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + match v { + "external" => Ok(LldMode::External), + "self-contained" => Ok(LldMode::SelfContained), + _ => Err(E::custom(&format!("unknown mode {}", v))), + } + } + } + + deserializer.deserialize_any(LldModeVisitor) + } +} diff --git a/standalonex/src/bootstrap/src/core/config/llvm.rs b/standalonex/src/bootstrap/src/core/config/llvm.rs new file mode 100644 index 00000000..872d46b7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/llvm.rs @@ -0,0 +1,39 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + optimize: Option = "optimize", + thin_lto: Option = "thin-lto", + release_debuginfo: Option = "release-debuginfo", + assertions: Option = "assertions", + tests: Option = "tests", + enzyme: Option = "enzyme", + plugins: Option = "plugins", + ccache: Option = "ccache", + static_libstdcpp: Option = "static-libstdcpp", + libzstd: Option = "libzstd", + ninja: Option = "ninja", + targets: Option = "targets", + experimental_targets: Option = "experimental-targets", + link_jobs: Option = "link-jobs", + link_shared: Option = "link-shared", + version_suffix: Option = "version-suffix", + clang_cl: Option = "clang-cl", + cflags: Option = "cflags", + cxxflags: Option = "cxxflags", + ldflags: Option = "ldflags", + use_libcxx: Option = "use-libcxx", + use_linker: Option = "use-linker", + allow_old_toolchain: Option = "allow-old-toolchain", + offload: Option = "offload", + polly: Option = "polly", + clang: Option = "clang", + enable_warnings: Option = "enable-warnings", + download_ci_llvm: Option = "download-ci-llvm", + build_config: Option> = "build-config", + enable_projects: Option = "enable-projects", + } +} + diff --git a/standalonex/src/bootstrap/src/core/config/llvm_lib_unwind.rs b/standalonex/src/bootstrap/src/core/config/llvm_lib_unwind.rs new file mode 100644 index 00000000..39d403ee --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/llvm_lib_unwind.rs @@ -0,0 +1,21 @@ +use crate::prelude::*; +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum LlvmLibunwind { + #[default] + No, + InTree, + System, +} + +impl FromStr for LlvmLibunwind { + type Err = String; + +fn from_str(value: &str) -> Result { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config/merge.rs b/standalonex/src/bootstrap/src/core/config/merge.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config/mod.rs b/standalonex/src/bootstrap/src/core/config/mod.rs index 9f09dd13..cb8ea9c2 100644 --- a/standalonex/src/bootstrap/src/core/config/mod.rs +++ b/standalonex/src/bootstrap/src/core/config/mod.rs @@ -1,7 +1,83 @@ -#[allow(clippy::module_inception)] -mod config; +use crate::prelude::*; + +pub mod build; +pub mod changeid; +pub mod ci; +pub mod ciconfig; +pub mod color; +pub mod config_base; +pub mod config_ci; +pub mod config_part2; +pub mod config_part3; +pub mod config_part4; + +pub mod config_part6; +pub mod config_part7; +pub mod config_toml; +pub mod config_types; +pub mod config_utils; +pub mod debug_info_level; +pub mod dist; +pub mod dry_run; pub mod flags; -#[cfg(test)] -mod tests; +pub mod install; +pub mod lld_mode; +pub mod llvm; +pub mod llvm_lib_unwind; +pub mod merge; +pub mod replaceop; +pub mod rust; +pub mod rust_optimize; +pub mod rustclto; +pub mod rustfmt; +pub mod splitdebuginfo; +pub mod string_or_int; +pub mod stringorbool; +pub mod subcommand; +pub use crate::core::config::subcommand::Subcommand; +pub mod target_selection; +pub mod target; +pub mod tomlconfig; +pub mod tomltarget; +pub mod warnings; + + +pub use target_selection::*; +pub use build::*; +pub use changeid::*; +pub use ci::*; +pub use ciconfig::*; +pub use color::*; +pub use config_base::*; +pub use config_ci::*; +pub use config_part2::*; +pub use config_part3::*; +pub use config_part4::*; -pub use config::*; +pub use config_part6::*; +pub use config_part7::*; +pub use config_toml::*; +pub use config_types::*; +pub use config_utils::*; +pub use debug_info_level::*; +pub use dist::*; +pub use dry_run::*; +pub use flags::*; +pub use install::*; +pub use llvm::*; +pub use llvm_lib_unwind::*; +pub use merge::*; +pub use replaceop::*; +pub use rust::*; +pub use rust_optimize::*; +pub use rustclto::*; +pub use rustfmt::*; +pub use splitdebuginfo::*; +pub use string_or_int::*; +pub use stringorbool::*; +pub use subcommand::*; +pub use target_selection::*; +pub use target::*; +pub use tomlconfig::*; +pub use tomltarget::*; +pub use warnings::*; diff --git a/standalonex/src/bootstrap/src/core/config/replaceop.rs b/standalonex/src/bootstrap/src/core/config/replaceop.rs new file mode 100644 index 00000000..3b34b9ec --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/replaceop.rs @@ -0,0 +1,10 @@ +/// Describes how to handle conflicts in merging two [`TomlConfig`] +#[derive(Copy, Clone, Debug)] +enum ReplaceOpt { + /// Silently ignore a duplicated value + IgnoreDuplicate, + /// Override the current value, even if it's `Some` + Override, + /// Exit with an error on duplicate values + ErrorOnDuplicate, +} diff --git a/standalonex/src/bootstrap/src/core/config/rust.rs b/standalonex/src/bootstrap/src/core/config/rust.rs new file mode 100644 index 00000000..64ca764b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/rust.rs @@ -0,0 +1,63 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option = "optimize", + debug: Option = "debug", + codegen_units: Option = "codegen-units", + codegen_units_std: Option = "codegen-units-std", + rustc_debug_assertions: Option = "debug-assertions", + randomize_layout: Option = "randomize-layout", + std_debug_assertions: Option = "debug-assertions-std", + overflow_checks: Option = "overflow-checks", + overflow_checks_std: Option = "overflow-checks-std", + debug_logging: Option = "debug-logging", + debuginfo_level: Option = "debuginfo-level", + debuginfo_level_rustc: Option = "debuginfo-level-rustc", + debuginfo_level_std: Option = "debuginfo-level-std", + debuginfo_level_tools: Option = "debuginfo-level-tools", + debuginfo_level_tests: Option = "debuginfo-level-tests", + backtrace: Option = "backtrace", + incremental: Option = "incremental", + parallel_compiler: Option = "parallel-compiler", + default_linker: Option = "default-linker", + channel: Option = "channel", + description: Option = "description", + musl_root: Option = "musl-root", + rpath: Option = "rpath", + strip: Option = "strip", + frame_pointers: Option = "frame-pointers", + stack_protector: Option = "stack-protector", + verbose_tests: Option = "verbose-tests", + optimize_tests: Option = "optimize-tests", + codegen_tests: Option = "codegen-tests", + omit_git_hash: Option = "omit-git-hash", + dist_src: Option = "dist-src", + save_toolstates: Option = "save-toolstates", + codegen_backends: Option> = "codegen-backends", + llvm_bitcode_linker: Option = "llvm-bitcode-linker", + lld: Option = "lld", + lld_mode: Option = "use-lld", + llvm_tools: Option = "llvm-tools", + deny_warnings: Option = "deny-warnings", + backtrace_on_ice: Option = "backtrace-on-ice", + verify_llvm_ir: Option = "verify-llvm-ir", + thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", + remap_debuginfo: Option = "remap-debuginfo", + jemalloc: Option = "jemalloc", + test_compare_mode: Option = "test-compare-mode", + llvm_libunwind: Option = "llvm-libunwind", + control_flow_guard: Option = "control-flow-guard", + ehcont_guard: Option = "ehcont-guard", + new_symbol_mangling: Option = "new-symbol-mangling", + profile_generate: Option = "profile-generate", + profile_use: Option = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option = "download-rustc", + lto: Option = "lto", + validate_mir_opts: Option = "validate-mir-opts", + std_features: Option> = "std-features", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/rust_optimize.rs b/standalonex/src/bootstrap/src/core/config/rust_optimize.rs new file mode 100644 index 00000000..36adbee9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/rust_optimize.rs @@ -0,0 +1,25 @@ +use crate::prelude::*; +use serde::Deserializer; +use crate::core::config::config_part6::OptimizeVisitor; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RustOptimize { + String(String), + Int(u8), + Bool(bool), +} + +impl Default for RustOptimize { +fn default() -> RustOptimize { + RustOptimize::Bool(false) + } +} + +impl<'de> Deserialize<'de> for RustOptimize { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} diff --git a/standalonex/src/bootstrap/src/core/config/rustclto.rs b/standalonex/src/bootstrap/src/core/config/rustclto.rs new file mode 100644 index 00000000..993e25d6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/rustclto.rs @@ -0,0 +1,24 @@ + +/// LTO mode used for compiling rustc itself. +#[derive(Default, Clone, PartialEq, Debug)] +pub enum RustcLto { + Off, + #[default] + ThinLocal, + Thin, + Fat, +} + +impl std::str::FromStr for RustcLto { + type Err = String; + +fn from_str(s: &str) -> Result { + match s { + "thin-local" => Ok(RustcLto::ThinLocal), + "thin" => Ok(RustcLto::Thin), + "fat" => Ok(RustcLto::Fat), + "off" => Ok(RustcLto::Off), + _ => Err(format!("Invalid value for rustc LTO: {s}")), + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config/rustfmt.rs b/standalonex/src/bootstrap/src/core/config/rustfmt.rs new file mode 100644 index 00000000..f346f645 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/rustfmt.rs @@ -0,0 +1,9 @@ +use crate::prelude::*; +#[derive(Clone, Debug, Default)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + #[default] + LazyEvaluated, +} diff --git a/standalonex/src/bootstrap/src/core/config/splitdebuginfo.rs b/standalonex/src/bootstrap/src/core/config/splitdebuginfo.rs new file mode 100644 index 00000000..c0f3b937 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/splitdebuginfo.rs @@ -0,0 +1,35 @@ +use crate::prelude::*; +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + #[default] + Off, +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + +fn from_str(s: &str) -> Result { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.example.toml`. +pub fn default_for_platform(target: TargetSelection) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.is_windows() { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config/string_or_int.rs b/standalonex/src/bootstrap/src/core/config/string_or_int.rs new file mode 100644 index 00000000..7de6d383 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/string_or_int.rs @@ -0,0 +1,8 @@ +use crate::prelude::*; + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum StringOrInt { + String(String), + Int(i64), +} diff --git a/standalonex/src/bootstrap/src/core/config/stringorbool.rs b/standalonex/src/bootstrap/src/core/config/stringorbool.rs new file mode 100644 index 00000000..b9b65a22 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/stringorbool.rs @@ -0,0 +1,19 @@ +use crate::prelude::*; +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { +fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +impl StringOrBool { +pub fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} diff --git a/standalonex/src/bootstrap/src/core/config/subcommand.rs b/standalonex/src/bootstrap/src/core/config/subcommand.rs new file mode 100644 index 00000000..3592d55e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/subcommand.rs @@ -0,0 +1,181 @@ +use crate::prelude::*; + +pub mod subcommand_groups; +use subcommand_groups::{QaTool, BuildTool, DistTool, MiscTool}; + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum Subcommand { + Qa(QaTool), + Build(BuildTool), + Dist(DistTool), + Misc(MiscTool), +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Qa(qa_tool) => match qa_tool { + QaTool::Bench { .. } => Kind::Bench, + QaTool::Check { .. } => Kind::Check, + QaTool::Clippy { .. } => Kind::Clippy, + QaTool::Fix { .. } => Kind::Fix, + QaTool::Format { .. } => Kind::Format, + QaTool::Test { .. } => Kind::Test, + QaTool::Miri { .. } => Kind::Miri, + QaTool::Suggest { .. } => Kind::Suggest, + QaTool::Perf { .. } => Kind::Perf, + }, + Subcommand::Build(build_tool) => match build_tool { + BuildTool::Build { .. } => Kind::Build, + BuildTool::Doc { .. } => Kind::Doc, + }, + Subcommand::Dist(dist_tool) => match dist_tool { + DistTool::Dist { .. } => Kind::Dist, + DistTool::Install { .. } => Kind::Install, + }, + Subcommand::Misc(misc_tool) => match misc_tool { + MiscTool::Clean { .. } => Kind::Clean, + MiscTool::Run { .. } => Kind::Run, + MiscTool::Setup { .. } => Kind::Setup, + MiscTool::Vendor { .. } => Kind::Vendor, + }, + } + } + + pub fn compiletest_rustc_args(&self) -> Vec<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref compiletest_rustc_args, .. }) => { + compiletest_rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + } + } + + pub fn fail_fast(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { no_fail_fast, .. }) | Subcommand::Qa(QaTool::Miri { no_fail_fast, .. }) => { + !no_fail_fast + } + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match self { + Subcommand::Qa(QaTool::Test { doc, no_doc, .. }) | Subcommand::Qa(QaTool::Miri { no_doc, doc, .. }) => { + if *doc { + DocTests::Only + } else if *no_doc { + DocTests::No + } else { + DocTests::Yes + } + } + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { bless, .. }) => *bless, + _ => false, + } + } + + pub fn extra_checks(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref extra_checks, .. }) => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + + pub fn only_modified(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { only_modified, .. }) => *only_modified, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { force_rerun, .. }) => *force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { rustfix_coverage, .. }) => *rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref compare_mode, .. }) => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref pass, .. }) => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref run, .. }) => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match self { + Subcommand::Build(BuildTool::Doc { open, .. }) => *open, + _ => false, + } + } + + pub fn json(&self) -> bool { + match self { + Subcommand::Build(BuildTool::Doc { json, .. }) => *json, + _ => false, + } + } + + pub fn vendor_versioned_dirs(&self) -> bool { + match self { + Subcommand::Misc(MiscTool::Vendor { versioned_dirs, .. }) => *versioned_dirs, + _ => false, + } + } + + pub fn vendor_sync_args(&self) -> Vec { + match self { + Subcommand::Misc(MiscTool::Vendor { sync, .. }) => sync.clone(), + _ => vec![], + } + } +} + +/// Returns the shell completion for a given shell, if the result differs from the current +/// content of `path`. If `path` does not exist, always returns `Some`. +pub fn get_completion(shell: G, path: &Path) -> Option { + let mut cmd = Flags::command(); + let current = if !path.exists() { + String::new() + } else { + std::fs::read_to_string(path).unwrap_or_else(|_| { + eprintln!("couldn't read {}", path.display()); + crate::exit!(1) + }) + }; + let mut buf = Vec::new(); + clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); + if buf == current.as_bytes() { + return None; + } + Some(String::from_utf8(buf).expect("completion script should be UTF-8")) +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config/subcommand_groups.rs b/standalonex/src/bootstrap/src/core/config/subcommand_groups.rs new file mode 100644 index 00000000..5226d214 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/subcommand_groups.rs @@ -0,0 +1,112 @@ +use crate::prelude::*; + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum QaTool { + Check { + #[arg(long)] + all_targets: bool, + }, + Clippy { + #[arg(long)] + fix: bool, + #[arg(long, requires = "fix")] + allow_dirty: bool, + #[arg(long, requires = "fix")] + allow_staged: bool, + #[arg(global = true, short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] + allow: Vec, + #[arg(global = true, short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] + deny: Vec, + #[arg(global = true, short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] + warn: Vec, + #[arg(global = true, short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] + forbid: Vec, + }, + Fix, + Format { + #[arg(long)] + check: bool, + #[arg(long)] + all: bool, + }, + Test { + #[arg(long)] + no_fail_fast: bool, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + test_args: Vec, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + compiletest_rustc_args: Vec, + #[arg(long)] + no_doc: bool, + #[arg(long)] + doc: bool, + #[arg(long)] + bless: bool, + #[arg(long, value_name = "check | build | run")] + pass: Option, + #[arg(long, value_name = "auto | always | never")] + run: Option, + #[arg(long)] + rustfix_coverage: bool, + }, + Miri { + #[arg(long)] + no_fail_fast: bool, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + test_args: Vec, + #[arg(long)] + no_doc: bool, + #[arg(long)] + doc: bool, + }, + Bench { + #[arg(long, allow_hyphen_values(true))] + test_args: Vec, + }, + Suggest { + #[arg(long)] + run: bool, + }, + Perf, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum BuildTool { + Build, + Doc { + #[arg(long)] + open: bool, + #[arg(long)] + json: bool, + }, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum DistTool { + Dist, + Install, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum MiscTool { + Clean { + #[arg(long)] + all: bool, + #[arg(long, value_name = "N")] + stage: Option, + }, + Run { + #[arg(long, allow_hyphen_values(true))] + args: Vec, + }, + Setup { + #[arg(value_name = "|hook|editor|link")] + profile: Option, + }, + Vendor { + #[arg(long)] + sync: Vec, + #[arg(long)] + versioned_dirs: bool, + }, +} diff --git a/standalonex/src/bootstrap/src/core/config/target.rs b/standalonex/src/bootstrap/src/core/config/target.rs new file mode 100644 index 00000000..a4a9ffba --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/target.rs @@ -0,0 +1,13 @@ +use crate::prelude::*; +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { + target.no_std = true; + } + if triple.contains("emscripten") { + target.runner = Some("node".into()); + } + target + } +} diff --git a/standalonex/src/bootstrap/src/core/config/target_selection.rs b/standalonex/src/bootstrap/src/core/config/target_selection.rs new file mode 100644 index 00000000..d8f11b5b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/target_selection.rs @@ -0,0 +1,148 @@ +use crate::prelude::*; + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. +// Making !Copy is highly nontrivial! +pub struct TargetSelection { + pub triple: Interned, + file: Option>, + synthetic: bool, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file, synthetic: false } + } + + pub fn create_synthetic(triple: &str, file: &str) -> Self { + Self { + triple: INTERNER.intern_str(triple), + file: Some(INTERNER.intern_str(file)), + synthetic: true, + } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } + + // See src/bootstrap/synthetic_targets.rs + pub fn is_synthetic(&self) -> bool { + self.synthetic + } + + pub fn is_msvc(&self) -> bool { + self.contains("msvc") + } + + pub fn is_windows(&self) -> bool { + self.contains("windows") + } + + pub fn is_windows_gnu(&self) -> bool { + self.ends_with("windows-gnu") + } + + /// Path to the file defining the custom target, if any. + pub fn filepath(&self) -> Option<&Path> { + self.file.as_ref().map(Path::new) + } +} + +impl fmt::Display for TargetSelection { +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({file})")?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { +pub fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl PartialEq<&str> for TargetSelection { +fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +// Targets are often used as directory names throughout bootstrap. +// This impl makes it more ergonomics to use them as such. +impl AsRef for TargetSelection { +fn as_ref(&self) -> &Path { + self.triple.as_ref() + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option, + pub llvm_has_rust_patches: Option, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option, + pub llvm_libunwind: Option, + pub cc: Option, + pub cxx: Option, + pub ar: Option, + pub ranlib: Option, + pub default_linker: Option, + pub linker: Option, + pub split_debuginfo: Option, + pub sanitizers: Option, + pub profiler: Option, + pub rpath: Option, + pub crt_static: Option, + pub musl_root: Option, + pub musl_libdir: Option, + pub wasi_root: Option, + pub qemu_rootfs: Option, + pub runner: Option, + pub no_std: bool, + pub codegen_backends: Option>, +} diff --git a/standalonex/src/bootstrap/src/core/config/tests.rs b/standalonex/src/bootstrap/src/core/config/tests.rs index 73d78724..00cd99ce 100644 --- a/standalonex/src/bootstrap/src/core/config/tests.rs +++ b/standalonex/src/bootstrap/src/core/config/tests.rs @@ -22,7 +22,7 @@ pub(crate) fn parse(config: &str) -> Config { #[test] #[ignore] -fn download_ci_llvm() { +pub fn download_ci_llvm() { let config = parse(""); let is_available = llvm::is_ci_llvm_available(&config, config.llvm_assertions); if is_available { @@ -56,8 +56,8 @@ fn download_ci_llvm() { // - https://github.com/rust-lang/rust/issues/109120 // - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 #[test] -fn detect_src_and_out() { - fn test(cfg: Config, build_dir: Option<&str>) { +pub fn detect_src_and_out() { +pub fn test(cfg: Config, build_dir: Option<&str>) { // This will bring absolute form of `src/bootstrap` path let current_dir = std::env::current_dir().unwrap(); @@ -108,12 +108,12 @@ fn detect_src_and_out() { } #[test] -fn clap_verify() { +pub fn clap_verify() { Flags::command().debug_assert(); } #[test] -fn override_toml() { +pub fn override_toml() { let config = Config::parse_inner( Flags::parse(&[ "check".to_owned(), @@ -208,7 +208,7 @@ runner = "x86_64-runner" #[test] #[should_panic] -fn override_toml_duplicate() { +pub fn override_toml_duplicate() { Config::parse_inner( Flags::parse(&[ "check".to_owned(), @@ -221,8 +221,8 @@ fn override_toml_duplicate() { } #[test] -fn profile_user_dist() { - fn get_toml(file: &Path) -> Result { +pub fn profile_user_dist() { +pub fn get_toml(file: &Path) -> Result { let contents = if file.ends_with("config.toml") || env::var_os("RUST_BOOTSTRAP_CONFIG").is_some() { "profile = \"user\"".to_owned() @@ -237,7 +237,7 @@ fn profile_user_dist() { } #[test] -fn rust_optimize() { +pub fn rust_optimize() { assert!(parse("").rust_optimize.is_release()); assert!(!parse("rust.optimize = false").rust_optimize.is_release()); assert!(parse("rust.optimize = true").rust_optimize.is_release()); @@ -250,12 +250,12 @@ fn rust_optimize() { #[test] #[should_panic] -fn invalid_rust_optimize() { +pub fn invalid_rust_optimize() { parse("rust.optimize = \"a\""); } #[test] -fn verify_file_integrity() { +pub fn verify_file_integrity() { let config = parse(""); let tempfile = config.tempdir().join(".tmp-test-file"); @@ -271,7 +271,7 @@ fn verify_file_integrity() { } #[test] -fn rust_lld() { +pub fn rust_lld() { assert!(matches!(parse("").lld_mode, LldMode::Unused)); assert!(matches!(parse("rust.use-lld = \"self-contained\"").lld_mode, LldMode::SelfContained)); assert!(matches!(parse("rust.use-lld = \"external\"").lld_mode, LldMode::External)); @@ -281,12 +281,12 @@ fn rust_lld() { #[test] #[should_panic] -fn parse_config_with_unknown_field() { +pub fn parse_config_with_unknown_field() { parse("unknown-key = 1"); } #[test] -fn parse_change_id_with_unknown_field() { +pub fn parse_change_id_with_unknown_field() { let config = r#" change-id = 3461 unknown-key = 1 @@ -297,7 +297,7 @@ fn parse_change_id_with_unknown_field() { } #[test] -fn order_of_clippy_rules() { +pub fn order_of_clippy_rules() { let args = vec![ "clippy".to_string(), "--fix".to_string(), @@ -329,7 +329,7 @@ fn order_of_clippy_rules() { } #[test] -fn clippy_rule_separate_prefix() { +pub fn clippy_rule_separate_prefix() { let args = vec!["clippy".to_string(), "-A clippy:all".to_string(), "-W clippy::style".to_string()]; let config = Config::parse(Flags::parse(&args)); @@ -347,7 +347,7 @@ fn clippy_rule_separate_prefix() { } #[test] -fn verbose_tests_default_value() { +pub fn verbose_tests_default_value() { let config = Config::parse(Flags::parse(&["build".into(), "compiler".into()])); assert_eq!(config.verbose_tests, false); @@ -356,7 +356,7 @@ fn verbose_tests_default_value() { } #[test] -fn parse_rust_std_features() { +pub fn parse_rust_std_features() { let config = parse("rust.std-features = [\"panic-unwind\", \"backtrace\"]"); let expected_features: BTreeSet = ["panic-unwind", "backtrace"].into_iter().map(|s| s.to_string()).collect(); @@ -364,7 +364,7 @@ fn parse_rust_std_features() { } #[test] -fn parse_rust_std_features_empty() { +pub fn parse_rust_std_features_empty() { let config = parse("rust.std-features = []"); let expected_features: BTreeSet = BTreeSet::new(); assert_eq!(config.rust_std_features, expected_features); @@ -372,17 +372,17 @@ fn parse_rust_std_features_empty() { #[test] #[should_panic] -fn parse_rust_std_features_invalid() { +pub fn parse_rust_std_features_invalid() { parse("rust.std-features = \"backtrace\""); } #[test] -fn parse_jobs() { +pub fn parse_jobs() { assert_eq!(parse("build.jobs = 1").jobs, Some(1)); } #[test] -fn jobs_precedence() { +pub fn jobs_precedence() { // `--jobs` should take precedence over using `--set build.jobs`. let config = Config::parse_inner( @@ -435,7 +435,7 @@ fn jobs_precedence() { } #[test] -fn check_rustc_if_unchanged_paths() { +pub fn check_rustc_if_unchanged_paths() { let config = parse(""); let normalised_allowed_paths: Vec<_> = RUSTC_IF_UNCHANGED_ALLOWED_PATHS .iter() diff --git a/standalonex/src/bootstrap/src/core/config/tomlconfig.rs b/standalonex/src/bootstrap/src/core/config/tomlconfig.rs new file mode 100644 index 00000000..ad1e6511 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/tomlconfig.rs @@ -0,0 +1,32 @@ +use crate::prelude::*; +/// Structure of the `config.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +pub(crate) struct Nix { + nixpkgs_path: Option, + rust_overlay_path: Option, + rust_bootstrap_nix_path: Option, + configuration_nix_path: Option, + rust_src_flake_path: Option, +} + +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct TomlConfig { + #[serde(flatten)] + change_id: ChangeIdWrapper, + build: Option, + install: Option, + llvm: Option, + rust: Option, + target: Option>, + dist: Option, + ci: Option, + nix: Option, + profile: Option, + stage0_path: Option, +} + diff --git a/standalonex/src/bootstrap/src/core/config/tomltarget.rs b/standalonex/src/bootstrap/src/core/config/tomltarget.rs new file mode 100644 index 00000000..7ff46ddd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/tomltarget.rs @@ -0,0 +1,30 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option = "cc", + cxx: Option = "cxx", + ar: Option = "ar", + ranlib: Option = "ranlib", + default_linker: Option = "default-linker", + linker: Option = "linker", + split_debuginfo: Option = "split-debuginfo", + llvm_config: Option = "llvm-config", + llvm_has_rust_patches: Option = "llvm-has-rust-patches", + llvm_filecheck: Option = "llvm-filecheck", + llvm_libunwind: Option = "llvm-libunwind", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + rpath: Option = "rpath", + crt_static: Option = "crt-static", + musl_root: Option = "musl-root", + musl_libdir: Option = "musl-libdir", + wasi_root: Option = "wasi-root", + qemu_rootfs: Option = "qemu-rootfs", + no_std: Option = "no-std", + codegen_backends: Option> = "codegen-backends", + runner: Option = "runner", + } +} diff --git a/standalonex/src/bootstrap/src/core/config/warnings.rs b/standalonex/src/bootstrap/src/core/config/warnings.rs new file mode 100644 index 00000000..966ee7b4 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config/warnings.rs @@ -0,0 +1,9 @@ +use crate::prelude::*; +/// Whether to deny warnings, emit them as warnings, or use the default behavior +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Warnings { + Deny, + Warn, + #[default] + Default, +} diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_core/Cargo.toml b/standalonex/src/bootstrap/src/core/config_crates/config_core/Cargo.toml new file mode 100644 index 00000000..a3bf3834 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_core/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "config_core" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_derive = "1.0" diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_core/src/lib.rs b/standalonex/src/bootstrap/src/core/config_crates/config_core/src/lib.rs new file mode 100644 index 00000000..f24b9e77 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_core/src/lib.rs @@ -0,0 +1,44 @@ +pub trait Merge { + fn merge(&mut self, other: Self, replace: ReplaceOpt); +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum ReplaceOpt { + IgnoreDuplicate, + Override, + ErrorOnDuplicate, +} + +impl Merge for Option +where + T: Merge + Sized, +{ + fn merge(&mut self, other: Self, replace: ReplaceOpt) { + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.is_none() { + *self = other; + } + } + ReplaceOpt::Override => { + if other.is_some() { + *self = other; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.is_some() { + if self.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option"); + std::process::exit(2); + } + } else { + *self = other; + } + } + } + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_macros/Cargo.toml b/standalonex/src/bootstrap/src/core/config_crates/config_macros/Cargo.toml new file mode 100644 index 00000000..e6855114 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_macros/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "config_macros" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "2.0", features = ["full", "extra-traits"] } +quote = "1.0" +proc-macro2 = "1.0" +config_core = { path = "../config_core" } \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_macros/src/lib.rs b/standalonex/src/bootstrap/src/core/config_crates/config_macros/src/lib.rs new file mode 100644 index 00000000..2da47ac0 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_macros/src/lib.rs @@ -0,0 +1,119 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::parse::{Parse, ParseStream, Result}; +use syn::{parse_macro_input, Ident, LitStr, Token}; +use syn::braced; + +struct ConfigInput { + attrs: Vec, + struct_token: Token![struct], + ident: Ident, + brace_token: syn::token::Brace, + fields: syn::punctuated::Punctuated, // Changed from ItemStruct to ConfigField +} + +struct ConfigField { + ident: Ident, + colon_token: syn::token::Colon, + ty: syn::Type, + eq_token: Option, + key: Option, +} + +impl Parse for ConfigInput { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(syn::Attribute::parse_outer)?; + let struct_token = input.parse()?; + let ident = input.parse()?; + let content; + let brace_token = braced!(content in input); + let fields = content.parse_terminated(ConfigField::parse, Token![,])?; + Ok(ConfigInput { + attrs, + struct_token, + ident, + brace_token, + fields, + }) + } +} + +impl Parse for ConfigField { + fn parse(input: ParseStream) -> Result { + let ident = input.parse()?; + let colon_token = input.parse()?; + let ty = input.parse()?; + let eq_token = input.parse().ok(); + let key = if eq_token.is_some() { + Some(input.parse()?) + } else { + None + }; + Ok(ConfigField { + ident, + colon_token, + ty, + eq_token, + key, + }) + } +} + +#[proc_macro] +pub fn define_config(input: TokenStream) -> TokenStream { + let ConfigInput { attrs, ident, fields, .. } = parse_macro_input!(input as ConfigInput); + + let field_names: Vec<_> = fields.iter().map(|f| &f.ident).collect(); + let field_types: Vec<_> = fields.iter().map(|f| &f.ty).collect(); + let field_keys: Vec<_> = fields.iter().map(|f| { + if let Some(key_lit) = &f.key { + quote! { #key_lit } + } else { + let ident_str = LitStr::new(&f.ident.to_string(), f.ident.span()); + quote! { #ident_str } + } + }).collect(); + + let expanded = quote! { + #(#attrs)* + pub struct #ident { + #(pub #field_names: #field_types,)* + } + + impl config_core::Merge for #ident { + fn merge(&mut self, other: Self, replace: config_core::ReplaceOpt) { + #( + match replace { + config_core::ReplaceOpt::IgnoreDuplicate => { + if self.#field_names.is_none() { + self.#field_names = other.#field_names; + } + }, + config_core::ReplaceOpt::Override => { + if other.#field_names.is_some() { + self.#field_names = other.#field_names; + } + } + config_core::ReplaceOpt::ErrorOnDuplicate => { + if other.#field_names.is_some() { + if self.#field_names.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option: `{}`", #field_keys); + panic!("overriding existing option"); + } + } + } else { + self.#field_names = other.#field_names; + } + } + } + )* + } + } + }; + expanded.into() +} diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.lock b/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.lock new file mode 100644 index 00000000..fae53d25 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.lock @@ -0,0 +1,94 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "config_core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "config_macros" +version = "0.1.0" +dependencies = [ + "config_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "config_tests" +version = "0.1.0" +dependencies = [ + "config_core", + "config_macros", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.toml b/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.toml new file mode 100644 index 00000000..b055dd81 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_tests/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "config_tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +config_core = { path = "../config_core" } +config_macros = { path = "../config_macros" } diff --git a/standalonex/src/bootstrap/src/core/config_crates/config_tests/src/main.rs b/standalonex/src/bootstrap/src/core/config_crates/config_tests/src/main.rs new file mode 100644 index 00000000..dd0c7ee2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_crates/config_tests/src/main.rs @@ -0,0 +1,92 @@ +use config_macros::define_config; +use config_core::Merge; + +define_config! { + struct MyConfig { + field1: Option = "field1", + field2: Option = "field2", + } +} + +define_config! { + #[derive(Clone)] + struct ComplexConfig { + name: Option = "name_key", + version: Option = "version_key", + enabled: Option = "enabled_key", + count: Option = "count_key", + path: Option = "path_key", + } +} + +fn main() { + // Test MyConfig + let mut config1 = MyConfig { + field1: Some("value1".to_string()), + field2: None, + }; + + let config2 = MyConfig { + field1: None, + field2: Some(123), + }; + + config1.merge(config2, config_core::ReplaceOpt::IgnoreDuplicate); + + assert_eq!(config1.field1, Some("value1".to_string())); + assert_eq!(config1.field2, Some(123)); + + println!("MyConfig tests passed!"); + + // Test ComplexConfig + let complex_config_a = ComplexConfig { + name: Some("AppA".to_string()), + version: Some(1.0), + enabled: Some(true), + count: None, + path: Some("/tmp/a".into()), + }; + + let complex_config_b = ComplexConfig { + name: Some("AppB".to_string()), + version: Some(2.0), + enabled: None, + count: Some(100), + path: None, + }; + + // Test ReplaceOpt::Override + let mut merged_override = complex_config_a.clone(); + merged_override.merge(complex_config_b.clone(), config_core::ReplaceOpt::Override); + assert_eq!(merged_override.name, Some("AppB".to_string())); + assert_eq!(merged_override.version, Some(2.0)); + assert_eq!(merged_override.enabled, Some(true)); + assert_eq!(merged_override.count, Some(100)); + assert_eq!(merged_override.path, Some("/tmp/a".into())); + + println!("ComplexConfig ReplaceOpt::Override tests passed!"); + + // Test ReplaceOpt::IgnoreDuplicate + let mut merged_ignore = complex_config_a.clone(); + merged_ignore.merge(complex_config_b.clone(), config_core::ReplaceOpt::IgnoreDuplicate); + assert_eq!(merged_ignore.name, Some("AppA".to_string())); + assert_eq!(merged_ignore.version, Some(1.0)); + assert_eq!(merged_ignore.enabled, Some(true)); + assert_eq!(merged_ignore.count, Some(100)); + assert_eq!(merged_ignore.path, Some("/tmp/a".into())); + + println!("ComplexConfig ReplaceOpt::IgnoreDuplicate tests passed!"); + + // Test ReplaceOpt::ErrorOnDuplicate (should panic if a duplicate is found) + let complex_config_c = complex_config_a.clone(); // Use a clone to move into the closure + let res = std::panic::catch_unwind(move || { + let mut merged_error = complex_config_c; + merged_error.merge(complex_config_b.clone(), config_core::ReplaceOpt::ErrorOnDuplicate); + }); + assert!(res.is_err()); + + println!("ComplexConfig ReplaceOpt::ErrorOnDuplicate tests passed (panic caught)! +"); + + println!("All tests passed!"); +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_processor/Cargo.toml b/standalonex/src/bootstrap/src/core/config_processor/Cargo.toml new file mode 100644 index 00000000..184abe58 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_processor/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "bootstrap-config-processor" +version = "0.1.0" +edition = "2021" + +[dependencies] +bootstrap-config-utils = { path = "../config_utils" } +# Placeholder for the main bootstrap crate dependency +# bootstrap = { path = "../../../../" } # Adjust path as needed diff --git a/standalonex/src/bootstrap/src/core/config_processor/src/lib.rs b/standalonex/src/bootstrap/src/core/config_processor/src/lib.rs new file mode 100644 index 00000000..8d2ad9bb --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_processor/src/lib.rs @@ -0,0 +1,110 @@ +use bootstrap_config_utils::parsed_config::ParsedConfig; +use bootstrap_config_utils::dry_run::DryRun; // Import DryRun + +// Placeholder for the actual bootstrap::Config struct +pub struct BootstrapConfig { + pub dry_run: bool, + pub out_dir: std::path::PathBuf, + pub channel: Option, + pub jobs: Option, + pub build_triple: Option, + pub rust_optimize_tests: Option, + pub docs: Option, + pub docs_minification: Option, + pub rust_rpath: Option, + pub rust_strip: Option, + pub rust_dist_src: Option, + pub deny_warnings: Option, + pub dist_include_mingw_linker: Option, + pub llvm_optimize: Option, + pub llvm_static_stdcpp: Option, + pub llvm_libzstd: Option, + pub llvm_assertions: Option, + pub llvm_tests: bool, + pub llvm_enzyme_flag: Option, + pub llvm_offload: bool, + pub llvm_plugins: bool, +} + +impl Default for BootstrapConfig { + fn default() -> Self { + BootstrapConfig { + dry_run: false, + out_dir: std::path::PathBuf::from("build"), + channel: None, + jobs: None, + build_triple: None, + rust_optimize_tests: None, + docs: None, + docs_minification: None, + rust_rpath: None, + rust_strip: None, + rust_dist_src: None, + deny_warnings: None, + dist_include_mingw_linker: None, + llvm_optimize: None, + llvm_static_stdcpp: None, + llvm_libzstd: None, + llvm_assertions: None, + llvm_tests: false, + llvm_enzyme_flag: None, + llvm_offload: false, + llvm_plugins: false, + } + } +} + +pub fn process_config(parsed_config: ParsedConfig) -> BootstrapConfig { + BootstrapConfig { + dry_run: parsed_config.dry_run != DryRun::Disabled, + out_dir: parsed_config.out, + channel: parsed_config.channel, + jobs: parsed_config.jobs, + build_triple: parsed_config.build_triple, + rust_optimize_tests: parsed_config.rust_optimize_tests, + docs: parsed_config.docs, + docs_minification: parsed_config.docs_minification, + rust_rpath: parsed_config.rust_rpath, + rust_strip: parsed_config.rust_strip, + rust_dist_src: parsed_config.rust_dist_src, + deny_warnings: parsed_config.deny_warnings, + dist_include_mingw_linker: parsed_config.dist_include_mingw_linker, + llvm_optimize: parsed_config.llvm_optimize, + llvm_static_stdcpp: parsed_config.llvm_static_stdcpp, + llvm_libzstd: parsed_config.llvm_libzstd, + llvm_assertions: parsed_config.llvm_assertions, + llvm_tests: parsed_config.llvm_tests, + llvm_enzyme_flag: parsed_config.llvm_enzyme_flag, + llvm_offload: parsed_config.llvm_offload, + llvm_plugins: parsed_config.llvm_plugins, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bootstrap_config_utils::parsed_config::ParsedConfig; + use bootstrap_config_utils::dry_run::DryRun; + use std::path::PathBuf; + + #[test] + fn test_process_config() { + let mut parsed_config = ParsedConfig::default(); + parsed_config.dry_run = DryRun::UserSelected; + parsed_config.out = PathBuf::from("/tmp/test_output"); + parsed_config.channel = Some("nightly".to_string()); + parsed_config.jobs = Some(8); + parsed_config.llvm_tests = true; + parsed_config.llvm_enzyme_flag = Some(true); + + let bootstrap_config = process_config(parsed_config); + + assert_eq!(bootstrap_config.dry_run, true); + assert_eq!(bootstrap_config.out_dir, PathBuf::from("/tmp/test_output")); + assert_eq!(bootstrap_config.channel, Some("nightly".to_string())); + assert_eq!(bootstrap_config.jobs, Some(8)); + assert_eq!(bootstrap_config.llvm_tests, true); + assert_eq!(bootstrap_config.llvm_enzyme_flag, Some(true)); + } +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/build.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/build.rs new file mode 100644 index 00000000..1a5e4e3f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/build.rs @@ -0,0 +1,59 @@ +use crate::prelude::*; + +use config_macros::define_config; + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option = "build", + src: Option = "src", + host: Option> = "host", + target: Option> = "target", + build_dir: Option = "build-dir", + cargo: Option = "cargo", + rustc: Option = "rustc", + rustfmt: Option = "rustfmt", + cargo_clippy: Option = "cargo-clippy", + docs: Option = "docs", + compiler_docs: Option = "compiler-docs", + library_docs_private_items: Option = "library-docs-private-items", + docs_minification: Option = "docs-minification", + submodules: Option = "submodules", + gdb: Option = "gdb", + lldb: Option = "lldb", + nodejs: Option = "nodejs", + npm: Option = "npm", + python: Option = "python", + reuse: Option = "reuse", + locked_deps: Option = "locked-deps", + vendor: Option = "vendor", + full_bootstrap: Option = "full-bootstrap", + bootstrap_cache_path: Option = "bootstrap-cache-path", + extended: Option = "extended", + tools: Option> = "tools", + verbose: Option = "verbose", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + cargo_native_static: Option = "cargo-native-static", + low_priority: Option = "low-priority", + configure_args: Option> = "configure-args", + local_rebuild: Option = "local-rebuild", + print_step_timings: Option = "print-step-timings", + print_step_rusage: Option = "print-step-rusage", + check_stage: Option = "check-stage", + doc_stage: Option = "doc-stage", + build_stage: Option = "build-stage", + test_stage: Option = "test-stage", + install_stage: Option = "install-stage", + dist_stage: Option = "dist-stage", + bench_stage: Option = "bench-stage", + patch_binaries_for_nix: Option = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally + metrics: Option = "metrics", + android_ndk: Option = "android-ndk", + optimized_compiler_builtins: Option = "optimized-compiler-builtins", + jobs: Option = "jobs", + compiletest_diff_tool: Option = "compiletest-diff-tool", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/changeid.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/changeid.rs new file mode 100644 index 00000000..37b0d273 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/changeid.rs @@ -0,0 +1,11 @@ +use crate::prelude::*; + +/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type +/// for the "change-id" field to parse it even if other fields are invalid. This ensures +/// that if deserialization fails due to other fields, we can still provide the changelogs +/// to allow developers to potentially find the reason for the failure in the logs.. +#[derive(Deserialize, Default)] +pub(crate) struct ChangeIdWrapper { + #[serde(alias = "change-id")] + pub(crate) inner: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/ci.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/ci.rs new file mode 100644 index 00000000..4046d5ac --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/ci.rs @@ -0,0 +1,13 @@ +use config_macros::define_config; + +define_config! { + /// TOML representation of CI-related paths and settings. + #[derive(Default)] + struct Ci { + channel_file: Option = "channel-file", + version_file: Option = "version-file", + tools_dir: Option = "tools-dir", + llvm_project_dir: Option = "llvm-project-dir", + gcc_dir: Option = "gcc-dir", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/ciconfig.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/ciconfig.rs new file mode 100644 index 00000000..a6da8270 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/ciconfig.rs @@ -0,0 +1,10 @@ +use crate::prelude::*; +/// Configuration for CI-related paths and settings. +#[derive(Debug, Default, Clone)] +pub struct CiConfig { + pub channel_file: PathBuf, + pub version_file: PathBuf, + pub tools_dir: PathBuf, + pub llvm_project_dir: PathBuf, + pub gcc_dir: PathBuf, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/color.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/color.rs new file mode 100644 index 00000000..74bd4f88 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/color.rs @@ -0,0 +1,8 @@ +use crate::prelude::*; +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Color { + Always, + Never, + #[default] + Auto, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config.rs new file mode 100644 index 00000000..e0af4300 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config.rs @@ -0,0 +1,162 @@ +//! Serialized configuration of a build. +//! +//! This module implements parsing `config.toml` configuration files to tweak +//! how the build runs. + +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fmt::{self, Display}; +use std::io::IsTerminal; +use std::path::{Path, PathBuf, absolute}; +use std::process::Command; +use std::str::FromStr; +use std::sync::OnceLock; +use std::{cmp, env, fs}; + +use build_helper::ci::CiEnv; +use build_helper::exit; +use build_helper::git::{GitConfig, get_closest_merge_commit, output_result}; +use serde::{Deserialize, Deserializer}; +use serde_derive::Deserialize; + +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::build_steps::llvm; +pub use crate::core::config::flags::Subcommand; +use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::core::download::is_download_ci_available; +use crate::utils::cache::{INTERNER, Interned}; +use crate::utils::channel::{self, GitInfo}; +use crate::utils::helpers::{self, exe, output, t}; + +/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. +/// This means they can be modified and changes to these paths should never trigger a compiler build +/// when "if-unchanged" is set. +/// +/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during +/// the diff check. +/// +/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build +/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. +/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the +/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. +#[rustfmt::skip] // We don't want rustfmt to oneline this list +pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ + ":!src/tools", + ":!tests", + ":!triagebot.toml", +]; + +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name).replace("_", "-") + ); + }; +} + + +pub use target_selection; + + + // use build_helper::ci::CiEnv; + // use build_helper::exit; + // use build_helper::git::get_closest_merge_commit; + // use build_helper::git::GitConfig; + // use build_helper::git::output_result; + // use cc::Build; + // use clap::builder::styling::Color; + // use clap::Command; + // use clap::Subcommand; + // use clap::ValueEnum; + // use cmake::Config; + // use crate::BTreeSet; + // use crate::Build; + // use crate::Cell; + // use crate::Command; + // use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; + // use crate::core::build_steps::llvm; + // use crate::core::build_steps::llvm::Llvm; + // use crate::core::build_steps::setup::Profile; + // use crate::core::build_steps::setup::Profile::Dist; + // use crate::core::build_steps::tool::LibcxxVersion::Llvm; + +use changeid::ChangeIdWrapper; +use ciconfig::CiConfig; +use color::Color; +use config_base::Config; +use debug_info_level::DebuginfoLevel; +use dry_run::BUILDER_CONFIG_FILENAME; +use rustclto::RustcLto; +use rustfmt::RustfmtState; +use rust_optimize::RustOptimize; +use splitdebuginfo::SplitDebuginfo; +use stringorbool::StringOrBool; +use subcommand::get_completion; +use subcommand::Subcommand; +use subcommand::Subcommand::Build; +use subcommand::Subcommand::Dist; +use subcommand::Subcommand::Install; +use tomlconfig::TomlConfig; +use warnings::Warnings; +use bootstrap_config_utils::default_opts; +use bootstrap_config_utils::get_builder_toml; +use bootstrap_config_utils::get_toml; + +// use crate::core::download::is_download_ci_available; + // use crate::define_config; + // use crate::Display; + // use crate::DocTests; +pub use dry_run::*; + // use crate::env; + // use crate::exe; + // use crate::exit; + // use crate::Flags; + // use crate::fs; + // use crate::GitInfo; + // use crate::GitRepo::Llvm; + // use crate::HashMap; + // use crate::HashSet; + // use crate::helpers; + // use crate::Kind; + // use crate::Kind::Build; + // use crate::Kind::Dist; + // use crate::Kind::Install; +pub use lld_mode::*; + // use crate::LlvmLibunwind; + // use crate::OnceLock; + // use crate::output; + // use crate::Path; + // use crate::PathBuf; + // use crate::RefCell; + // use crate::str::FromStr; + // use crate::t; + // use crate::Target; +pub use target_selection::TargetSelection; + // use crate::utils::cache::Interned; + // use crate::utils::cache::INTERNER; + // use crate::utils::channel; + // use crate::utils::shared_helpers::exe; + // use crate::utils::tarball::OverlayKind::Llvm; + // use crate::utils::tarball::OverlayKind::Rust; + // use serde_derive::Deserialize; + // use serde::Deserialize; + // use serde::Deserializer; + // use std::cell::Cell; + // use std::cell::RefCell; + // use std::cmp; + // use std::collections::BTreeSet; + // use std::collections::HashMap; + // use std::collections::HashSet; + // use std::env; + // use std::fmt; + // use std::fmt::Display; + // use std::fs; + // use std::path::absolute; + // use std::path::Path; + // use std::path::PathBuf; + // use std::process::Command; + // use std::str::FromStr; + // use std::sync::OnceLock; + // use termcolor::Color; diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_base.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_base.rs new file mode 100644 index 00000000..9ba86f8d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_base.rs @@ -0,0 +1,231 @@ +use crate::prelude::*; + +/// Global configuration for the entire build and/or bootstrap. +/// +/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. +/// +/// Note that this structure is not decoded directly into, but rather it is +/// filled out from the decoded forms of the structs below. For documentation +/// each field, see the corresponding fields in +/// `config.example.toml`. +#[derive(Default, Clone)] +pub struct Config { + pub change_id: Option, + pub bypass_bootstrap_lock: bool, + pub ccache: Option, + /// Call Build::ninja() instead of this. + pub ninja_in_file: bool, + pub verbose: usize, + pub submodules: Option, + pub compiler_docs: bool, + pub library_docs_private_items: bool, + pub docs_minification: bool, + pub docs: bool, + pub locked_deps: bool, + pub vendor: bool, + pub target_config: HashMap, + pub full_bootstrap: bool, + pub bootstrap_cache_path: Option, + pub extended: bool, + pub tools: Option>, + pub sanitizers: bool, + pub profiler: bool, + pub omit_git_hash: bool, + pub skip: Vec, + pub include_default_paths: bool, + pub rustc_error_format: Option, + pub json_output: bool, + pub test_compare_mode: bool, + pub color: Color, + pub patch_binaries_for_nix: Option, + pub stage0_path: Option, + pub stage0_metadata: build_helper::stage0_parser::Stage0, + pub android_ndk: Option, + /// Whether to use the `c` feature of the `compiler_builtins` crate. + pub optimized_compiler_builtins: bool, + + pub stdout_is_tty: bool, + pub stderr_is_tty: bool, + + pub on_fail: Option, + pub stage: u32, + pub keep_stage: Vec, + pub keep_stage_std: Vec, + pub src: PathBuf, + /// defaults to `config.toml` + pub config: Option, + pub jobs: Option, + pub cmd: Subcommand, + pub incremental: bool, + pub dry_run: DryRun, + pub dump_bootstrap_shims: bool, + /// Arguments appearing after `--` to be forwarded to tools, + /// e.g. `--fix-broken` or test arguments. + pub free_args: Vec, + + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option, + #[cfg(test)] + pub download_rustc_commit: Option, + + pub deny_warnings: bool, + pub backtrace_on_ice: bool, + + // llvm codegen options + pub llvm_assertions: bool, + pub llvm_tests: bool, + pub llvm_enzyme: bool, + pub llvm_offload: bool, + pub llvm_plugins: bool, + pub llvm_optimize: bool, + pub llvm_thin_lto: bool, + pub llvm_release_debuginfo: bool, + pub llvm_static_stdcpp: bool, + pub llvm_libzstd: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell>, + #[cfg(test)] + pub llvm_link_shared: Cell>, + pub llvm_clang_cl: Option, + pub llvm_targets: Option, + pub llvm_experimental_targets: Option, + pub llvm_link_jobs: Option, + pub llvm_version_suffix: Option, + pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_enable_warnings: bool, + pub llvm_from_ci: bool, + pub llvm_build_config: HashMap, + pub llvm_enable_projects: Option, + + pub lld_mode: LldMode, + pub lld_enabled: bool, + pub llvm_tools_enabled: bool, + pub llvm_bitcode_linker_enabled: bool, + + pub llvm_cflags: Option, + pub llvm_cxxflags: Option, + pub llvm_ldflags: Option, + pub llvm_use_libcxx: bool, + + // rust codegen options + pub rust_optimize: RustOptimize, + pub rust_codegen_units: Option, + pub rust_codegen_units_std: Option, + + pub rustc_debug_assertions: bool, + pub std_debug_assertions: bool, + + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: DebuginfoLevel, + pub rust_debuginfo_level_std: DebuginfoLevel, + pub rust_debuginfo_level_tools: DebuginfoLevel, + pub rust_debuginfo_level_tests: DebuginfoLevel, + pub rust_rpath: bool, + pub rust_strip: bool, + pub rust_frame_pointers: bool, + pub rust_stack_protector: Option, + pub rustc_default_linker: Option, + pub rust_optimize_tests: bool, + pub rust_dist_src: bool, + pub rust_codegen_backends: Vec, + pub rust_verify_llvm_ir: bool, + pub rust_thin_lto_import_instr_limit: Option, + pub rust_randomize_layout: bool, + pub rust_remap_debuginfo: bool, + pub rust_new_symbol_mangling: Option, + pub rust_profile_use: Option, + pub rust_profile_generate: Option, + pub rust_lto: RustcLto, + pub rust_validate_mir_opts: Option, + pub rust_std_features: BTreeSet, + pub llvm_profile_use: Option, + pub llvm_profile_generate: bool, + pub llvm_libunwind_default: Option, + pub enable_bolt_settings: bool, + + pub reproducible_artifacts: Vec, + + pub build: TargetSelection, + pub hosts: Vec, + pub targets: Vec, + pub local_rebuild: bool, + pub jemalloc: bool, + pub control_flow_guard: bool, + pub ehcont_guard: bool, + + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_compression_formats: Option>, + pub dist_compression_profile: String, + pub dist_include_mingw_linker: bool, + pub dist_vendor: bool, + + // libstd features + pub backtrace: bool, // support for RUST_BACKTRACE + + // misc + pub low_priority: bool, + pub channel: String, + pub description: Option, + pub verbose_tests: bool, + pub save_toolstates: Option, + pub print_step_timings: bool, + pub print_step_rusage: bool, + + // Fallback musl-root for all targets + pub musl_root: Option, + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: PathBuf, + pub libdir: Option, + pub mandir: Option, + pub codegen_tests: bool, + pub nodejs: Option, + pub npm: Option, + pub gdb: Option, + pub lldb: Option, + pub python: Option, + pub reuse: Option, + pub cargo_native_static: bool, + pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, + + pub cargo_info: channel::GitInfo, + pub rust_analyzer_info: channel::GitInfo, + pub clippy_info: channel::GitInfo, + pub miri_info: channel::GitInfo, + pub rustfmt_info: channel::GitInfo, + pub enzyme_info: channel::GitInfo, + pub in_tree_llvm_info: channel::GitInfo, + pub in_tree_gcc_info: channel::GitInfo, + + // These are either the stage0 downloaded binaries or the locally installed ones. + pub initial_cargo: PathBuf, + pub initial_rustc: PathBuf, + pub initial_cargo_clippy: Option, + + #[cfg(not(test))] + initial_rustfmt: RefCell, + #[cfg(test)] + pub initial_rustfmt: RefCell, + + pub ci: CiConfig, + + /// The paths to work with. For example: with `./x check foo bar` we get + /// `paths=["foo", "bar"]`. + pub paths: Vec, + + /// Command for visual diff display, e.g. `diff-tool --color=always`. + pub compiletest_diff_tool: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_ci.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_ci.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_part2.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part2.rs new file mode 100644 index 00000000..f143513d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part2.rs @@ -0,0 +1,148 @@ +use crate::prelude::*; +/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. +/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. +pub fn check_incompatible_options_for_ci_rustc( + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `rust.{}` is incompatible with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `rust.{}` has no effect with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let (Some(current_rust_config), Some(ci_rust_config)) = + (current_config_toml.rust, ci_config_toml.rust) + else { + return Ok(()); + }; + + let Rust { + // Following options are the CI rustc incompatible ones. + optimize, + randomize_layout, + debug_logging, + debuginfo_level_rustc, + llvm_tools, + llvm_bitcode_linker, + lto, + stack_protector, + strip, + lld_mode, + jemalloc, + rpath, + channel, + description, + incremental, + default_linker, + std_features, + + // Rest of the options can simply be ignored. + debug: _, + codegen_units: _, + codegen_units_std: _, + rustc_debug_assertions: _, + std_debug_assertions: _, + overflow_checks: _, + overflow_checks_std: _, + debuginfo_level: _, + debuginfo_level_std: _, + debuginfo_level_tools: _, + debuginfo_level_tests: _, + backtrace: _, + parallel_compiler: _, + musl_root: _, + verbose_tests: _, + optimize_tests: _, + codegen_tests: _, + omit_git_hash: _, + dist_src: _, + save_toolstates: _, + codegen_backends: _, + lld: _, + deny_warnings: _, + backtrace_on_ice: _, + verify_llvm_ir: _, + thin_lto_import_instr_limit: _, + remap_debuginfo: _, + test_compare_mode: _, + llvm_libunwind: _, + control_flow_guard: _, + ehcont_guard: _, + new_symbol_mangling: _, + profile_generate: _, + profile_use: _, + download_rustc: _, + validate_mir_opts: _, + frame_pointers: _, + } = ci_rust_config; + + // There are two kinds of checks for CI rustc incompatible options: + // 1. Checking an option that may change the compiler behaviour/output. + // 2. Checking an option that have no effect on the compiler behaviour/output. + // + // If the option belongs to the first category, we call `err` macro for a hard error; + // otherwise, we just print a warning with `warn` macro. + + err!(current_rust_config.optimize, optimize); + err!(current_rust_config.randomize_layout, randomize_layout); + err!(current_rust_config.debug_logging, debug_logging); + err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc); + err!(current_rust_config.rpath, rpath); + err!(current_rust_config.strip, strip); + err!(current_rust_config.lld_mode, lld_mode); + err!(current_rust_config.llvm_tools, llvm_tools); + err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker); + err!(current_rust_config.jemalloc, jemalloc); + err!(current_rust_config.default_linker, default_linker); + err!(current_rust_config.stack_protector, stack_protector); + err!(current_rust_config.lto, lto); + err!(current_rust_config.std_features, std_features); + + warn!(current_rust_config.channel, channel); + warn!(current_rust_config.description, description); + warn!(current_rust_config.incremental, incremental); + + Ok(()) +} + +pub fn set(field: &mut T, val: Option) { + if let Some(v) = val { + *field = v; + } +} + +pub fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_part3.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part3.rs new file mode 100644 index 00000000..fd757152 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part3.rs @@ -0,0 +1,107 @@ +use crate::prelude::*; +/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. +/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. +#[cfg(not(feature = "bootstrap-self-test"))] +pub(crate) fn check_incompatible_options_for_ci_llvm( + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let (Some(current_llvm_config), Some(ci_llvm_config)) = + (current_config_toml.llvm, ci_config_toml.llvm) + else { + return Ok(()); + }; + + let Llvm { + optimize, + thin_lto, + release_debuginfo, + assertions: _, + tests: _, + plugins, + ccache: _, + static_libstdcpp: _, + libzstd, + ninja: _, + targets, + experimental_targets, + link_jobs: _, + link_shared: _, + version_suffix, + clang_cl, + cflags, + cxxflags, + ldflags, + use_libcxx, + use_linker, + allow_old_toolchain, + offload, + polly, + clang, + enable_warnings, + download_ci_llvm: _, + build_config, + enzyme, + enable_projects: _, + } = ci_llvm_config; + + err!(current_llvm_config.optimize, optimize); + err!(current_llvm_config.thin_lto, thin_lto); + err!(current_llvm_config.release_debuginfo, release_debuginfo); + err!(current_llvm_config.libzstd, libzstd); + err!(current_llvm_config.targets, targets); + err!(current_llvm_config.experimental_targets, experimental_targets); + err!(current_llvm_config.clang_cl, clang_cl); + err!(current_llvm_config.version_suffix, version_suffix); + err!(current_llvm_config.cflags, cflags); + err!(current_llvm_config.cxxflags, cxxflags); + err!(current_llvm_config.ldflags, ldflags); + err!(current_llvm_config.use_libcxx, use_libcxx); + err!(current_llvm_config.use_linker, use_linker); + err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); + err!(current_llvm_config.offload, offload); + err!(current_llvm_config.polly, polly); + err!(current_llvm_config.clang, clang); + err!(current_llvm_config.build_config, build_config); + err!(current_llvm_config.plugins, plugins); + err!(current_llvm_config.enzyme, enzyme); + + warn!(current_llvm_config.enable_warnings, enable_warnings); + + Ok(()) +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_part4.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part4.rs new file mode 100644 index 00000000..d2d346bd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part4.rs @@ -0,0 +1,1622 @@ +use crate::prelude::*; +use std::path::absolute; +impl Config { + + + + + pub(crate) fn parse_inner( + mut flags: Flags, + get_toml: impl Fn(&Path) -> Result, + ) -> Config { + let mut config = Config::default_opts(); + + // Set flags. + config.paths = std::mem::take(&mut flags.paths); + config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); + config.include_default_paths = flags.include_default_paths; + config.rustc_error_format = flags.rustc_error_format; + config.json_output = flags.json_output; + config.on_fail = flags.on_fail; + config.cmd = flags.cmd; + config.incremental = flags.incremental; + config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + config.dump_bootstrap_shims = flags.dump_bootstrap_shims; + config.keep_stage = flags.keep_stage; + config.keep_stage_std = flags.keep_stage_std; + config.color = flags.color; + config.free_args = std::mem::take(&mut flags.free_args); + config.llvm_profile_use = flags.llvm_profile_use; + config.llvm_profile_generate = flags.llvm_profile_generate; + config.enable_bolt_settings = flags.enable_bolt_settings; + config.bypass_bootstrap_lock = flags.bypass_bootstrap_lock; + + // Infer the rest of the configuration. + + config.src = if let Some(src) = flags.src { + src + } else if let Some(src) = build_src_from_toml { + src + } else { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + manifest_dir.parent().unwrap().parent().unwrap().to_owned() + }; + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + + config.stage0_metadata = build_helper::stage0_parser::parse_stage0_file( + &toml.stage0_path.expect("stage0_path must be set"), + ); + + // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + let toml_path = flags + .config + .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = config.src.join(toml_path); + } + + let file_content = t!(fs::read_to_string(&config.ci.channel_file)); + let ci_channel = file_content.trim_end(); + + // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + // but not if `config.toml` hasn't been created. + let mut toml = if !using_default_path || toml_path.exists() { + config.config = Some(if cfg!(not(feature = "bootstrap-self-test")) { + toml_path.canonicalize().unwrap() + } else { + toml_path.clone() + }); + get_toml::get_toml(&toml_path).unwrap_or_else(|e| { + eprintln!("ERROR: Failed to parse '{}': {e}", toml_path.display()); + exit!(2); + }) + } else { + config.config = None; + TomlConfig::default() + }; + + if cfg!(test) { + // When configuring bootstrap for tests, make sure to set the rustc and Cargo to the + // same ones used to call the tests (if custom ones are not defined in the toml). If we + // don't do that, bootstrap will use its own detection logic to find a suitable rustc + // and Cargo, which doesn't work when the caller is specÃŦfying a custom local rustc or + // Cargo in their config.toml. + let build = toml.build.get_or_insert_with(Default::default); + build.rustc = build.rustc.take().or(std::env::var_os("RUSTC").map(|p| p.into())); + build.cargo = build.cargo.take().or(std::env::var_os("CARGO").map(|p| p.into())); + } + + if let Some(include) = &toml.profile { + // Allows creating alias for profile names, allowing + // profiles to be renamed while maintaining back compatibility + // Keep in sync with `profile_aliases` in bootstrap.py + let profile_aliases = HashMap::from([("user", "dist")]); + let include = match profile_aliases.get(include.as_str()) { + Some(alias) => alias, + None => include.as_str(), + }; + let mut include_path = config.src.clone(); + include_path.push("src"); + include_path.push("bootstrap"); + include_path.push("defaults"); + include_path.push(format!("config.{include}.toml")); + let included_toml = get_toml(&include_path).unwrap_or_else(|e| { + eprintln!( + "ERROR: Failed to parse default config profile at '{}': {e}", + include_path.display() + ); + exit!(2); + }); + toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); + } + + let mut override_toml = TomlConfig::default(); + for option in flags.set.iter() { +pub fn get_table(option: &str) -> Result { + toml::from_str(option).and_then(|table: toml::Value| TomlConfig::deserialize(table)) + } + + let mut err = match get_table(option) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => e, + }; + // We want to be able to set string values without quotes, + // like in `configure.py`. Try adding quotes around the right hand side + if let Some((key, value)) = option.split_once('=') { + if !value.contains('"') { + match get_table(&format!(r#"{key}="{value}""#)) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => err = e, + } + } + } + eprintln!("failed to parse override `{option}`: `{err}"); + exit!(2) + } + toml.merge(override_toml, ReplaceOpt::Override); + + let build_src = toml.build.as_ref().and_then(|b| b.src.clone()); + + let Ci { + channel_file, + version_file, + tools_dir, + llvm_project_dir, + gcc_dir, + } = toml.ci.unwrap_or_default(); + + set(&mut config.ci.channel_file, channel_file.map(PathBuf::from)); + set(&mut config.ci.version_file, version_file.map(PathBuf::from)); + set(&mut config.ci.tools_dir, tools_dir.map(PathBuf::from)); + set(&mut config.ci.llvm_project_dir, llvm_project_dir.map(PathBuf::from)); + set(&mut config.ci.gcc_dir, gcc_dir.map(PathBuf::from)); + + config.change_id = toml.change_id.inner; + + let Build { + build, + host, + target, + build_dir, + cargo, + rustc, + rustfmt, + cargo_clippy, + docs, + compiler_docs, + library_docs_private_items, + docs_minification, + submodules, + gdb, + lldb, + nodejs, + npm, + python, + reuse, + locked_deps, + vendor, + full_bootstrap, + bootstrap_cache_path, + extended, + tools, + verbose, + sanitizers, + profiler, + cargo_native_static, + low_priority, + configure_args, + local_rebuild, + print_step_timings, + print_step_rusage, + check_stage, + doc_stage, + build_stage, + test_stage, + install_stage, + dist_stage, + bench_stage, + patch_binaries_for_nix, + // This field is only used by bootstrap.py + metrics: _, + android_ndk, + optimized_compiler_builtins, + jobs, + compiletest_diff_tool, + src: build_src_from_toml, + } = toml.build.unwrap_or_default(); + + config.jobs = Some(threads_from_config(flags.jobs.unwrap_or(jobs.unwrap_or(0)))); + + if let Some(file_build) = build { + config.build = TargetSelection::from_user(&file_build); + }; + + set(&mut config.out, flags.build_dir.or_else(|| build_dir.map(PathBuf::from))); + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = absolute(&config.out).expect("can't make empty path absolute"); + } + + if cargo_clippy.is_some() && rustc.is_none() { + println!( + "WARNING: Using `build.cargo-clippy` without `build.rustc` usually fails due to toolchain conflict." + ); + } + + config.initial_cargo_clippy = cargo_clippy; + + config.initial_rustc = if let Some(rustc) = rustc { + if !flags.skip_stage0_validation { + config.check_stage0_version(&rustc, "rustc"); + } + rustc + } else { + config.download_beta_toolchain(); + config + .out + .join(config.build) + .join("stage0") + .join("bin") + .join(exe("rustc", config.build)) + }; + + config.initial_cargo = if let Some(cargo) = cargo { + if !flags.skip_stage0_validation { + config.check_stage0_version(&cargo, "cargo"); + } + cargo + } else { + config.download_beta_toolchain(); + config + .out + .join(config.build) + .join("stage0") + .join("bin") + .join(exe("cargo", config.build)) + }; + + // NOTE: it's important this comes *after* we set `initial_rustc` just above. + if config.dry_run { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { + arg_host + } else if let Some(file_host) = host { + file_host.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + vec![config.build] + }; + config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { + arg_target + } else if let Some(file_target) = target { + file_target.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + // If target is *not* configured, then default to the host + // toolchains. + config.hosts.clone() + }; + + config.nodejs = nodejs.map(PathBuf::from); + config.npm = npm.map(PathBuf::from); + config.gdb = gdb.map(PathBuf::from); + config.lldb = lldb.map(PathBuf::from); + config.python = python.map(PathBuf::from); + config.reuse = reuse.map(PathBuf::from); + config.submodules = submodules; + config.android_ndk = android_ndk; + config.bootstrap_cache_path = bootstrap_cache_path; + set(&mut config.low_priority, low_priority); + set(&mut config.compiler_docs, compiler_docs); + set(&mut config.library_docs_private_items, library_docs_private_items); + set(&mut config.docs_minification, docs_minification); + set(&mut config.docs, docs); + set(&mut config.locked_deps, locked_deps); + set(&mut config.vendor, vendor); + set(&mut config.full_bootstrap, full_bootstrap); + set(&mut config.extended, extended); + config.tools = tools; + set(&mut config.verbose, verbose); + set(&mut config.sanitizers, sanitizers); + set(&mut config.profiler, profiler); + set(&mut config.cargo_native_static, cargo_native_static); + set(&mut config.configure_args, configure_args); + set(&mut config.local_rebuild, local_rebuild); + set(&mut config.print_step_timings, print_step_timings); + set(&mut config.print_step_rusage, print_step_rusage); + config.patch_binaries_for_nix = patch_binaries_for_nix; + + config.verbose = cmp::max(config.verbose, flags.verbose as usize); + + // Verbose flag is a good default for `rust.verbose-tests`. + config.verbose_tests = config.is_verbose(); + + if let Some(install) = toml.install { + let Install { prefix, sysconfdir, docdir, bindir, libdir, mandir, datadir } = install; + config.prefix = prefix.map(PathBuf::from); + config.sysconfdir = sysconfdir.map(PathBuf::from); + config.datadir = datadir.map(PathBuf::from); + config.docdir = docdir.map(PathBuf::from); + // Handle bindir specifically, as it's not an Option in Config + if let Some(b) = bindir { + config.bindir = PathBuf::from(b); + } else if let Some(p) = &config.prefix { + config.bindir = p.join("bin"); + } + config.libdir = libdir.map(PathBuf::from); + config.mandir = mandir.map(PathBuf::from); + } + + config.llvm_assertions = + toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false)); + + // Store off these values as options because if they're not provided + // we'll infer default values for them later + let mut llvm_tests = None; + let mut llvm_enzyme = None; + let mut llvm_offload = None; + let mut llvm_plugins = None; + let mut debug = None; + let mut rustc_debug_assertions = None; + let mut std_debug_assertions = None; + let mut overflow_checks = None; + let mut overflow_checks_std = None; + let mut debug_logging = None; + let mut debuginfo_level = None; + let mut debuginfo_level_rustc = None; + let mut debuginfo_level_std = None; + let mut debuginfo_level_tools = None; + let mut debuginfo_level_tests = None; + let mut optimize = None; + let mut lld_enabled = None; + let mut std_features = None; + + let is_user_configured_rust_channel = + if let Some(channel) = toml.rust.as_ref().and_then(|r| r.channel.clone()) { + config.channel = channel; + true + } else { + false + }; + + let default = config.channel == "dev"; + config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(default); + + config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); // config.src is still the overall source root + config.cargo_info = GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("cargo")); + config.rust_analyzer_info = + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("rust-analyzer")); + config.clippy_info = + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("clippy")); + config.miri_info = GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("miri")); + config.rustfmt_info = + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("rustfmt")); + config.enzyme_info = + GitInfo::new(config.omit_git_hash, &config.ci.tools_dir.join("enzyme")); + config.in_tree_llvm_info = GitInfo::new(false, &config.ci.llvm_project_dir); + config.in_tree_gcc_info = GitInfo::new(false, &config.ci.gcc_dir); + + if let Some(rust) = toml.rust { + let Rust { + optimize: optimize_toml, + debug: debug_toml, + codegen_units, + codegen_units_std, + rustc_debug_assertions: rustc_debug_assertions_toml, + std_debug_assertions: std_debug_assertions_toml, + overflow_checks: overflow_checks_toml, + overflow_checks_std: overflow_checks_std_toml, + debug_logging: debug_logging_toml, + debuginfo_level: debuginfo_level_toml, + debuginfo_level_rustc: debuginfo_level_rustc_toml, + debuginfo_level_std: debuginfo_level_std_toml, + debuginfo_level_tools: debuginfo_level_tools_toml, + debuginfo_level_tests: debuginfo_level_tests_toml, + backtrace, + incremental, + parallel_compiler, + randomize_layout, + default_linker, + channel: _, // already handled above + description, + musl_root, + rpath, + verbose_tests, + optimize_tests, + codegen_tests, + omit_git_hash: _, // already handled above + dist_src, + save_toolstates, + codegen_backends, + lld: lld_enabled_toml, + llvm_tools, + llvm_bitcode_linker, + deny_warnings, + backtrace_on_ice, + verify_llvm_ir, + thin_lto_import_instr_limit, + remap_debuginfo, + jemalloc, + test_compare_mode, + llvm_libunwind, + control_flow_guard, + ehcont_guard, + new_symbol_mangling, + profile_generate, + profile_use, + download_rustc, + lto, + validate_mir_opts, + frame_pointers, + stack_protector, + strip, + lld_mode, + std_features: std_features_toml, + } = rust; + + config.download_rustc_commit = + config.download_ci_rustc_commit(download_rustc, config.llvm_assertions); + + debug = debug_toml; + rustc_debug_assertions = rustc_debug_assertions_toml; + std_debug_assertions = std_debug_assertions_toml; + overflow_checks = overflow_checks_toml; + overflow_checks_std = overflow_checks_std_toml; + debug_logging = debug_logging_toml; + debuginfo_level = debuginfo_level_toml; + debuginfo_level_rustc = debuginfo_level_rustc_toml; + debuginfo_level_std = debuginfo_level_std_toml; + debuginfo_level_tools = debuginfo_level_tools_toml; + debuginfo_level_tests = debuginfo_level_tests_toml; + lld_enabled = lld_enabled_toml; + std_features = std_features_toml; + + optimize = optimize_toml; + config.rust_new_symbol_mangling = new_symbol_mangling; + set(&mut config.rust_optimize_tests, optimize_tests); + set(&mut config.codegen_tests, codegen_tests); + set(&mut config.rust_rpath, rpath); + set(&mut config.rust_strip, strip); + set(&mut config.rust_frame_pointers, frame_pointers); + config.rust_stack_protector = stack_protector; + set(&mut config.jemalloc, jemalloc); + set(&mut config.test_compare_mode, test_compare_mode); + set(&mut config.backtrace, backtrace); + config.description = description; + set(&mut config.rust_dist_src, dist_src); + set(&mut config.verbose_tests, verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = incremental { + config.incremental = true; + } + set(&mut config.lld_mode, lld_mode); + set(&mut config.llvm_bitcode_linker_enabled, llvm_bitcode_linker); + + config.rust_randomize_layout = randomize_layout.unwrap_or_default(); + config.llvm_tools_enabled = llvm_tools.unwrap_or(true); + + // FIXME: Remove this option at the end of 2024. + if parallel_compiler.is_some() { + println!( + "WARNING: The `rust.parallel-compiler` option is deprecated and does nothing. The parallel compiler (with one thread) is now the default" + ); + } + + config.llvm_enzyme = + llvm_enzyme.unwrap_or(config.channel == "dev" || config.channel == "nightly"); + config.rustc_default_linker = default_linker; + config.musl_root = musl_root.map(PathBuf::from); + config.save_toolstates = save_toolstates.map(PathBuf::from); + set(&mut config.deny_warnings, match flags.warnings { + Warnings::Deny => Some(true), + Warnings::Warn => Some(false), + Warnings::Default => deny_warnings, + }); + set(&mut config.backtrace_on_ice, backtrace_on_ice); + set(&mut config.rust_verify_llvm_ir, verify_llvm_ir); + config.rust_thin_lto_import_instr_limit = thin_lto_import_instr_limit; + set(&mut config.rust_remap_debuginfo, remap_debuginfo); + set(&mut config.control_flow_guard, control_flow_guard); + set(&mut config.ehcont_guard, ehcont_guard); + config.llvm_libunwind_default = + llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = codegen_backends { + let available_backends = ["llvm", "cranelift", "gcc"]; + + config.rust_codegen_backends = backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + s.clone() + }).collect(); + } + + config.rust_codegen_units = codegen_units.map(threads_from_config); + config.rust_codegen_units_std = codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(profile_generate); + config.rust_lto = + lto.as_deref().map(|value| RustcLto::from_str(value).unwrap()).unwrap_or_default(); + config.rust_validate_mir_opts = validate_mir_opts; + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; + } + + config.reproducible_artifacts = flags.reproducible_artifact; + + // We need to override `rust.channel` if it's manually specified when using the CI rustc. + // This is because if the compiler uses a different channel than the one specified in config.toml, + // tests may fail due to using a different channel than the one used by the compiler during tests. + if let Some(commit) = &config.download_rustc_commit { + if is_user_configured_rust_channel { + println!( + "WARNING: `rust.download-rustc` is enabled. The `rust.channel` option will be overridden by the CI rustc's channel." + ); + + let channel = config + .read_file_by_commit(&config.ci.channel_file, commit) + .trim() + .to_owned(); + + config.channel = channel; + } + } else if config.rust_info.is_from_tarball() && !is_user_configured_rust_channel { + ci_channel.clone_into(&mut config.channel); + } + + if let Some(llvm) = toml.llvm { + let Llvm { + optimize: optimize_toml, + thin_lto, + release_debuginfo, + assertions: _, + tests, + enzyme, + plugins, + ccache, + static_libstdcpp, + libzstd, + ninja, + targets, + experimental_targets, + link_jobs, + link_shared, + version_suffix, + clang_cl, + cflags, + cxxflags, + ldflags, + use_libcxx, + use_linker, + allow_old_toolchain, + offload, + polly, + clang, + enable_warnings, + download_ci_llvm, + build_config, + enable_projects, + } = llvm; + match ccache { + Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), + Some(StringOrBool::Bool(true)) => { + config.ccache = Some("ccache".to_string()); + } + Some(StringOrBool::Bool(false)) | None => {} + } + set(&mut config.ninja_in_file, ninja); + llvm_tests = tests; + llvm_enzyme = enzyme; + llvm_offload = offload; + llvm_plugins = plugins; + set(&mut config.llvm_optimize, optimize_toml); + set(&mut config.llvm_thin_lto, thin_lto); + set(&mut config.llvm_release_debuginfo, release_debuginfo); + set(&mut config.llvm_static_stdcpp, static_libstdcpp); + set(&mut config.llvm_libzstd, libzstd); + if let Some(v) = link_shared { + config.llvm_link_shared.set(Some(v)); + } + + config.llvm_targets.clone_from(&targets); + config.llvm_experimental_targets.clone_from(&experimental_targets); + config.llvm_link_jobs = link_jobs; + config.llvm_version_suffix.clone_from(&version_suffix); + config.llvm_clang_cl.clone_from(&clang_cl); + config.llvm_enable_projects.clone_from(&enable_projects); + + config.llvm_cflags.clone_from(&cflags); + config.llvm_cxxflags.clone_from(&cxxflags); + config.llvm_ldflags.clone_from(&ldflags); + set(&mut config.llvm_use_libcxx, use_libcxx); + config.llvm_use_linker.clone_from(&use_linker); + config.llvm_allow_old_toolchain = allow_old_toolchain.unwrap_or(false); + config.llvm_offload = offload.unwrap_or(false); + config.llvm_polly = polly.unwrap_or(false); + config.llvm_clang = clang.unwrap_or(false); + config.llvm_enable_warnings = enable_warnings.unwrap_or(false); + config.llvm_build_config = build_config.clone().unwrap_or(Default::default()); + + config.llvm_from_ci = + config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions); + + if config.llvm_from_ci { + let warn = |option: &str| { + println!( + "WARNING: `{option}` will only be used on `compiler/rustc_llvm` build, not for the LLVM build." + ); + println!( + "HELP: To use `{option}` for LLVM builds, set `download-ci-llvm` option to false." + ); + }; + + if static_libstdcpp.is_some() { + warn("static-libstdcpp"); + } + + if link_shared.is_some() { + warn("link-shared"); + } + + // FIXME(#129153): instead of all the ad-hoc `download-ci-llvm` checks that follow, + // use the `builder-config` present in tarballs since #128822 to compare the local + // config to the ones used to build the LLVM artifacts on CI, and only notify users + // if they've chosen a different value. + + if libzstd.is_some() { + println!( + "WARNING: when using `download-ci-llvm`, the local `llvm.libzstd` option, \ + like almost all `llvm.*` options, will be ignored and set by the LLVM CI \ + artifacts builder config." + ); + println!( + "HELP: To use `llvm.libzstd` for LLVM/LLD builds, set `download-ci-llvm` option to false." + ); + } + } + + if !config.llvm_from_ci && config.llvm_thin_lto && link_shared.is_none() { + // If we're building with ThinLTO on, by default we want to link + // to LLVM shared, to avoid re-doing ThinLTO (which happens in + // the link step) with each stage. + config.llvm_link_shared.set(Some(true)); + } + } else { + config.llvm_from_ci = config.parse_download_ci_llvm(None, false); + } + + if let Some(t) = toml.target { + for (triple, cfg) in t { + let mut target = Target::from_triple(&triple); + + if let Some(ref s) = cfg.llvm_config { + if config.download_rustc_commit.is_some() && triple == *config.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } + target.llvm_config = Some(config.src.join(s)); + } + if let Some(patches) = cfg.llvm_has_rust_patches { + assert!( + config.submodules == Some(false) || cfg.llvm_config.is_some(), + "use of `llvm-has-rust-patches` is restricted to cases where either submodules are disabled or llvm-config been provided" + ); + target.llvm_has_rust_patches = Some(patches); + } + if let Some(ref s) = cfg.llvm_filecheck { + target.llvm_filecheck = Some(config.src.join(s)); + } + target.llvm_libunwind = cfg.llvm_libunwind.as_ref().map(|v| { + v.parse().unwrap_or_else(|_| { + panic!("failed to parse target.{triple}.llvm-libunwind") + }) + }); + if let Some(s) = cfg.no_std { + target.no_std = s; + } + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); + target.crt_static = cfg.crt_static; + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); + target.runner = cfg.runner; + target.sanitizers = cfg.sanitizers; + target.profiler = cfg.profiler; + target.rpath = cfg.rpath; + + if let Some(ref backends) = cfg.codegen_backends { + let available_backends = ["llvm", "cranelift", "gcc"]; + + target.codegen_backends = Some(backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + s.clone() + }).collect()); + } + + target.split_debuginfo = cfg.split_debuginfo.as_ref().map(|v| { + v.parse().unwrap_or_else(|_| { + panic!("invalid value for target.{triple}.split-debuginfo") + }) + }); + + config.target_config.insert(TargetSelection::from_user(&triple), target); + } + } + + if config.llvm_from_ci { + let triple = &config.build.triple; + let ci_llvm_bin = config.ci_llvm_root().join("bin"); + let build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + + if let Some(dist) = toml.dist { + let Dist { + sign_folder, + upload_addr, + src_tarball, + compression_formats, + compression_profile, + include_mingw_linker, + vendor, + } = dist; + config.dist_sign_folder = sign_folder.map(PathBuf::from); + config.dist_upload_addr = upload_addr; + config.dist_compression_formats = compression_formats; + set(&mut config.dist_compression_profile, compression_profile); + set(&mut config.rust_dist_src, src_tarball); + set(&mut config.dist_include_mingw_linker, include_mingw_linker); + config.dist_vendor = vendor.unwrap_or_else(|| { + // If we're building from git or tarball sources, enable it by default. + config.rust_info.is_managed_git_subrepository() + || config.rust_info.is_from_tarball() + }); + } + + if let Some(r) = rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } + + // Now that we've reached the end of our configuration, infer the + // default values for all options that we haven't otherwise stored yet. + + config.llvm_tests = llvm_tests.unwrap_or(false); + config.llvm_enzyme = llvm_enzyme.unwrap_or(false); + config.llvm_offload = llvm_offload.unwrap_or(false); + config.llvm_plugins = llvm_plugins.unwrap_or(false); + config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); + + // We make `x86_64-unknown-linux-gnu` use the self-contained linker by default, so we will + // build our internal lld and use it as the default linker, by setting the `rust.lld` config + // to true by default: + // - on the `x86_64-unknown-linux-gnu` target + // - on the `dev` and `nightly` channels + // - when building our in-tree llvm (i.e. the target has not set an `llvm-config`), so that + // we're also able to build the corresponding lld + // - or when using an external llvm that's downloaded from CI, which also contains our prebuilt + // lld + // - otherwise, we'd be using an external llvm, and lld would not necessarily available and + // thus, disabled + // - similarly, lld will not be built nor used by default when explicitly asked not to, e.g. + // when the config sets `rust.lld = false` + if config.build.triple == "x86_64-unknown-linux-gnu" + && config.hosts == [config.build] + && (config.channel == "dev" || config.channel == "nightly") + { + let no_llvm_config = config + .target_config + .get(&config.build) + .is_some_and(|target_config| target_config.llvm_config.is_none()); + let enable_lld = config.llvm_from_ci || no_llvm_config; + // Prefer the config setting in case an explicit opt-out is needed. + config.lld_enabled = lld_enabled.unwrap_or(enable_lld); + } else { + set(&mut config.lld_enabled, lld_enabled); + } + + if matches!(config.lld_mode, LldMode::SelfContained) + && !config.lld_enabled + && flags.stage.unwrap_or(0) > 0 + { + panic!( + "Trying to use self-contained lld as a linker, but LLD is not being added to the sysroot. Enable it with rust.lld = true." + ); + } + + let default_std_features = BTreeSet::from([String::from("panic-unwind")]); + config.rust_std_features = std_features.unwrap_or(default_std_features); + + let default = debug == Some(true); + config.rustc_debug_assertions = rustc_debug_assertions.unwrap_or(default); + config.std_debug_assertions = std_debug_assertions.unwrap_or(config.rustc_debug_assertions); + config.rust_overflow_checks = overflow_checks.unwrap_or(default); + config.rust_overflow_checks_std = + overflow_checks_std.unwrap_or(config.rust_overflow_checks); + + config.rust_debug_logging = debug_logging.unwrap_or(config.rustc_debug_assertions); + + let with_defaults = |debuginfo_level_specific: Option<_>| { + debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { + DebuginfoLevel::Limited + } else { + DebuginfoLevel::None + }) + }; + config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); + config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); + config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); + config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); + config.optimized_compiler_builtins = + optimized_compiler_builtins.unwrap_or(config.channel != "dev"); + config.compiletest_diff_tool = compiletest_diff_tool; + + let download_rustc = config.download_rustc_commit.is_some(); + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Check { .. } => flags.stage.or(check_stage).unwrap_or(0), + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags.stage.or(doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) + } + Subcommand::Build { .. } => { + flags.stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } | Subcommand::Miri { .. } => { + flags.stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags.stage.or(bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(install_stage).unwrap_or(2), + Subcommand::Perf { .. } => flags.stage.unwrap_or(1), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } + | Subcommand::Vendor { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && build_helper::ci::CiEnv::is_ci() { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Miri { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } + | Subcommand::Vendor { .. } + | Subcommand::Perf { .. } => {} + } + } + + config + } + + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + + pub(crate) fn test_args(&self) -> Vec<&str> { + let mut test_args = match self.cmd { + Subcommand::Test { ref test_args, .. } + | Subcommand::Bench { ref test_args, .. } + | Subcommand::Miri { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + test_args.extend(self.free_args.iter().map(|s| s.as_str())); + test_args + } + + pub(crate) fn args(&self) -> Vec<&str> { + let mut args = match self.cmd { + Subcommand::Run { ref args, .. } => { + args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + args.extend(self.free_args.iter().map(|s| s.as_str())); + args + } + + /// Returns the content of the given file at a specific commit. + pub(crate) fn read_file_by_commit(&self, file: &Path, commit: &str) -> String { + assert!( + self.rust_info.is_managed_git_subrepository(), + "`Config::read_file_by_commit` is not supported in non-git sources." + ); + + let mut git = helpers::git(Some(&self.src)); + git.arg("show").arg(format!("{commit}:{}", file.to_str().unwrap())); + output(git.as_command_mut()) + } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + pub(crate) fn artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = if self.rust_info.is_managed_git_subrepository() { + let channel = self + .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) + .trim() + .to_owned(); + let version = + self.read_file_by_commit(&self.ci.version_file, commit).trim().to_owned(); + (channel, version) + } else { + let channel = fs::read_to_string(&self.ci.channel_file); + let version = fs::read_to_string(&self.ci.version_file); + match (channel, version) { + (Ok(channel), Ok(version)) => { + (channel.trim().to_owned(), version.trim().to_owned()) + } + (channel, version) => { + let src = self.src.display(); + eprintln!("ERROR: failed to determine artifact channel and/or version"); + eprintln!( + "HELP: consider using a git checkout or ensure these files are readable" + ); + if let Err(channel) = channel { + eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); + } + if let Err(version) = version { + eprintln!("reading {src}/src/version failed: {version:?}"); + } + panic!(); + } + } + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } + + /// Try to find the relative path of `bindir`, otherwise return it in full. + pub fn bindir_relative(&self) -> &Path { + let bindir = &self.bindir; + if bindir.is_absolute() { + // Try to make it relative to the prefix. + if let Some(prefix) = &self.prefix { + if let Ok(stripped) = bindir.strip_prefix(prefix) { + return stripped; + } + } + } + bindir + } + + /// Try to find the relative path of `libdir`. + pub fn libdir_relative(&self) -> Option<&Path> { + let libdir = self.libdir.as_ref()?; + if libdir.is_relative() { + Some(libdir) + } else { + // Try to make it relative to the prefix. + libdir.strip_prefix(self.prefix.as_ref()?).ok() + } + } + + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(self.build).join("ci-llvm") + } + + /// Directory where the extracted `rustc-dev` component is stored. + pub(crate) fn ci_rustc_dir(&self) -> PathBuf { + assert!(self.download_rustc()); + self.out.join(self.build).join("ci-rustc") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(&self) -> bool { + let mut opt = self.llvm_link_shared.get(); + if opt.is_none() && self.dry_run { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if self.llvm_from_ci { + self.maybe_download_ci_llvm(); + let ci_llvm = self.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + self.llvm_link_shared.set(opt); + llvm_link_shared + } + + /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. + pub(crate) fn download_rustc(&self) -> bool { + self.download_rustc_commit().is_some() + } + + pub(crate) fn download_rustc_commit(&self) -> Option<&str> { + static DOWNLOAD_RUSTC: OnceLock> = OnceLock::new(); + if self.dry_run && DOWNLOAD_RUSTC.get().is_none() { + // avoid trying to actually download the commit + return self.download_rustc_commit.as_deref(); + } + + DOWNLOAD_RUSTC + .get_or_init(|| match &self.download_rustc_commit { + None => None, + Some(commit) => { + self.download_ci_rustc(commit); + + // CI-rustc can't be used without CI-LLVM. If `self.llvm_from_ci` is false, it means the "if-unchanged" + // logic has detected some changes in the LLVM submodule (download-ci-llvm=false can't happen here as + // we don't allow it while parsing the configuration). + if !self.llvm_from_ci { + // This happens when LLVM submodule is updated in CI, we should disable ci-rustc without an error + // to not break CI. For non-CI environments, we should return an error. + if CiEnv::is_ci() { + println!("WARNING: LLVM submodule has changes, `download-rustc` will be disabled."); + return None; + } else { + panic!("ERROR: LLVM submodule has changes, `download-rustc` can't be used."); + } + } + + if let Some(config_path) = &self.config { + let ci_config_toml = match get_builder_toml::get_builder_toml(self, "ci-rustc") { + Ok(ci_config_toml) => ci_config_toml, + Err(e) if e.to_string().contains("unknown field") => { + println!("WARNING: CI rustc has some fields that are no longer supported in bootstrap; download-rustc will be disabled."); + println!("HELP: Consider rebasing to a newer commit if available."); + return None; + }, + Err(e) => { + eprintln!("ERROR: Failed to parse CI rustc config.toml: {e}"); + exit!(2); + }, + }; + + let current_config_toml = get_toml::get_toml(config_path).unwrap(); + + // Check the config compatibility + // FIXME: this doesn't cover `--set` flags yet. + let res = check_incompatible_options_for_ci_rustc( + current_config_toml, + ci_config_toml, + ); + + // Primarily used by CI runners to avoid handling download-rustc incompatible + // options one by one on shell scripts. + let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE") + .is_some_and(|s| s == "1" || s == "true"); + + if disable_ci_rustc_if_incompatible && res.is_err() { + println!("WARNING: download-rustc is disabled with `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` env."); + return None; + } + + res.unwrap(); + } + + Some(commit.clone()) + } + }) + .as_deref() + } + + pub(crate) fn initial_rustfmt(&self) -> Option { + match &mut *self.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if self.dry_run { + return Some(PathBuf::new()); + } + let path = self.maybe_download_rustfmt(); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + + /// Runs a function if verbosity is greater than 0 + pub fn verbose(&self, f: impl Fn()) { + if self.is_verbose() { + f() + } + } + + pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).and_then(|t| t.sanitizers).unwrap_or(self.sanitizers) + } + + pub fn needs_sanitizer_runtime_built(&self, target: TargetSelection) -> bool { + // MSVC uses the Microsoft-provided sanitizer runtime, but all other runtimes we build. + !target.is_msvc() && self.sanitizers_enabled(target) + } + + pub fn any_sanitizers_to_build(&self) -> bool { + self.target_config + .iter() + .any(|(ts, t)| !ts.is_msvc() && t.sanitizers.unwrap_or(self.sanitizers)) + } + + pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { + match self.target_config.get(&target)?.profiler.as_ref()? { + StringOrBool::String(s) => Some(s), + StringOrBool::Bool(_) => None, + } + } + + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { + self.target_config + .get(&target) + .and_then(|t| t.profiler.as_ref()) + .map(StringOrBool::is_string_or_true) + .unwrap_or(self.profiler) + } + + pub fn any_profiler_enabled(&self) -> bool { + self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) + || self.profiler + } + + pub fn rpath_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).and_then(|t| t.rpath).unwrap_or(self.rust_rpath) + } + + pub fn llvm_enabled(&self, target: TargetSelection) -> bool { + self.codegen_backends(target).contains(&"llvm".to_owned()) + } + + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { + self.target_config + .get(&target) + .and_then(|t| t.llvm_libunwind) + .or(self.llvm_libunwind_default) + .unwrap_or(if target.contains("fuchsia") { + LlvmLibunwind::InTree + } else { + LlvmLibunwind::No + }) + } + + pub fn split_debuginfo(&self, target: TargetSelection) -> SplitDebuginfo { + self.target_config + .get(&target) + .and_then(|t| t.split_debuginfo) + .unwrap_or_else(|| SplitDebuginfo::default_for_platform(target)) + } + + /// Returns whether or not submodules should be managed by bootstrap. + pub fn submodules(&self) -> bool { + // If not specified in config, the default is to only manage + // submodules if we're currently inside a git repository. + self.submodules.unwrap_or(self.rust_info.is_managed_git_subrepository()) + } + + pub fn codegen_backends(&self, target: TargetSelection) -> &[String] { + self.target_config + .get(&target) + .and_then(|cfg| cfg.codegen_backends.as_deref()) + .unwrap_or(&self.rust_codegen_backends) + } + + pub fn default_codegen_backend(&self, target: TargetSelection) -> Option { + self.codegen_backends(target).first().cloned() + } + + pub fn git_config(&self) -> GitConfig<'_> { + GitConfig { + git_repository: &self.stage0_metadata.config.git_repository, + nightly_branch: &self.stage0_metadata.config.nightly_branch, + git_merge_commit_email: &self.stage0_metadata.config.git_merge_commit_email, + } + } + + /// Given a path to the directory of a submodule, update it. + /// + /// `relative_path` should be relative to the root of the git repository, not an absolute path. + /// + /// This *does not* update the submodule if `config.toml` explicitly says + /// not to, or if we're not in a git repository (like a plain source + /// tarball). Typically [`crate::Build::require_submodule`] should be + /// used instead to provide a nice error to the user if the submodule is + /// missing. + pub(crate) fn update_submodule(&self, relative_path: &str) { + if !self.submodules() { + return; + } + + let absolute_path = self.src.join(relative_path); + + // NOTE: The check for the empty directory is here because when running x.py the first time, + // the submodule won't be checked out. Check it out now so we can build it. + if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + && !helpers::dir_is_empty(&absolute_path) + { + return; + } + + // Submodule updating actually happens during in the dry run mode. We need to make sure that + // all the git commands below are actually executed, because some follow-up code + // in bootstrap might depend on the submodules being checked out. Furthermore, not all + // the command executions below work with an empty output (produced during dry run). + // Therefore, all commands below are marked with `run_always()`, so that they also run in + // dry run mode. + let submodule_git = || { + let mut cmd = helpers::git(Some(&absolute_path)); + cmd.run_always(); + cmd + }; + + // Determine commit checked out in submodule. + let checked_out_hash = output(submodule_git().args(["rev-parse", "HEAD"]).as_command_mut()); + let checked_out_hash = checked_out_hash.trim_end(); + // Determine commit that the submodule *should* have. + let recorded = output( + helpers::git(Some(&self.src)) + .run_always() + .args(["ls-tree", "HEAD"]) + .arg(relative_path) + .as_command_mut(), + ); + + let actual_hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + if actual_hash == checked_out_hash { + // already checked out + return; + } + + println!("Updating submodule {relative_path}"); + self.check_run( + helpers::git(Some(&self.src)) + .run_always() + .args(["submodule", "-q", "sync"]) + .arg(relative_path), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, + // even though that has no relation to the upstream for the submodule. + let current_branch = output_result( + helpers::git(Some(&self.src)) + .allow_failure() + .run_always() + .args(["symbolic-ref", "--short", "HEAD"]) + .as_command_mut(), + ) + .map(|b| b.trim().to_owned()); + + let mut git = helpers::git(Some(&self.src)).allow_failure(); + git.run_always(); + if let Ok(branch) = current_branch { + // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. + // This syntax isn't accepted by `branch.{branch}`. Strip it. + let branch = branch.strip_prefix("heads/").unwrap_or(&branch); + git.arg("-c").arg(format!("branch.{branch}.remote=origin")); + } + git.args(["submodule", "update", "--init", "--recursive", "--depth=1"]); + if progress { + git.arg("--progress"); + } + git.arg(relative_path); + git + }; + if !self.check_run(&mut update(true)) { + self.check_run(&mut update(false)); + } + + // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). + // diff-index reports the modifications through the exit status + let has_local_modifications = !self.check_run(submodule_git().allow_failure().args([ + "diff-index", + "--quiet", + "HEAD", + ])); + if has_local_modifications { + self.check_run(submodule_git().args(["stash", "push"])); + } + + self.check_run(submodule_git().args(["reset", "-q", "--hard"])); + self.check_run(submodule_git().args(["clean", "-qdfx"])); + + if has_local_modifications { + self.check_run(submodule_git().args(["stash", "pop"])); + } + } + + #[cfg(feature = "bootstrap-self-test")] + pub fn check_stage0_version(&self, _program_path: &Path, _component_name: &'static str) {} + + /// check rustc/cargo version is same or lower with 1 apart from the building one + #[cfg(not(feature = "bootstrap-self-test"))] + pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { + use build_helper::util::fail; + + if self.dry_run { + return; + } + + let stage0_output = output(Command::new(program_path).arg("--version")); + let mut stage0_output = stage0_output.lines().next().unwrap().split(' '); + + let stage0_name = stage0_output.next().unwrap(); + if stage0_name != component_name { + fail(&format!( + "Expected to find {component_name} at {} but it claims to be {stage0_name}", + program_path.display() + )); + } + + let stage0_version = + semver::Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) + .unwrap(); + let source_version = semver::Version::parse( + fs::read_to_string(self.src.join("src/version")).unwrap().trim(), + ) + .unwrap(); + if !(source_version == stage0_version + || (source_version.major == stage0_version.major + && (source_version.minor == stage0_version.minor + || source_version.minor == stage0_version.minor + 1))) + { + let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); + fail(&format!( + "Unexpected {component_name} version: {stage0_version}, we should use {prev_version}/{source_version} to build source with {source_version}" + )); + } + } + + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. +pub fn download_ci_rustc_commit( + &self, + download_rustc: Option, + llvm_assertions: bool, + ) -> Option { + if !is_download_ci_available(&self.build.triple, llvm_assertions) { + return None; + } + + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None => self.rust_info.is_managed_git_subrepository(), + Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => { + if !self.rust_info.is_managed_git_subrepository() { + println!( + "ERROR: `download-rustc=if-unchanged` is only compatible with Git managed sources." + ); + crate::exit!(1); + } + + true + } + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {other}") + } + }; + + // RUSTC_IF_UNCHANGED_ALLOWED_PATHS + let mut allowed_paths = RUSTC_IF_UNCHANGED_ALLOWED_PATHS.to_vec(); + + // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, allow + // these changes to speed up the build process for library developers. This provides consistent + // functionality for library developers between `download-rustc=true` and `download-rustc="if-unchanged"` + // options. + if !CiEnv::is_ci() { + allowed_paths.push(":!library"); + } + + let commit = if self.rust_info.is_managed_git_subrepository() { + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + match self.last_modified_commit(&allowed_paths, "download-rustc", if_unchanged) { + Some(commit) => commit, + None => { + if if_unchanged { + return None; + } + println!("ERROR: could not find commit hash for downloading rustc"); + println!("HELP: maybe your repository history is too shallow?"); + println!("HELP: consider setting `rust.download-rustc=false` in config.toml"); + println!("HELP: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + } + } else { + channel::read_commit_info_file(&self.src) + .map(|info| info.sha.trim().to_owned()) + .expect("git-commit-info is missing in the project root") + }; + + if CiEnv::is_ci() && { + let head_sha = + output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut()); + let head_sha = head_sha.trim(); + commit == head_sha + } { + eprintln!("CI rustc commit matches with HEAD and we are in CI."); + eprintln!( + "`rustc.download-ci` functionality will be skipped as artifacts are not available." + ); + return None; + } + + Some(commit) + } + +pub fn parse_download_ci_llvm( + &self, + download_ci_llvm: Option, + asserts: bool, + ) -> bool { + let download_ci_llvm = download_ci_llvm.unwrap_or(StringOrBool::Bool(true)); + + let if_unchanged = || { + if self.rust_info.is_from_tarball() { + // Git is needed for running "if-unchanged" logic. + println!("ERROR: 'if-unchanged' is only compatible with Git managed sources."); + crate::exit!(1); + } + + // Fetching the LLVM submodule is unnecessary for self-tests. + #[cfg(not(feature = "bootstrap-self-test"))] + self.update_submodule("src/llvm-project"); + + // Check for untracked changes in `src/llvm-project`. + let has_changes = self + .last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true) + .is_none(); + + // Return false if there are untracked changes, otherwise check if CI LLVM is available. + if has_changes { false } else { llvm::is_ci_llvm_available(self, asserts) } + }; + + match download_ci_llvm { + StringOrBool::Bool(b) => { + if !b && self.download_rustc_commit.is_some() { + panic!( + "`llvm.download-ci-llvm` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`." + ); + } + + // If download-ci-llvm=true we also want to check that CI llvm is available + b && llvm::is_ci_llvm_available(self, asserts) + } + StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(), + StringOrBool::String(other) => { + panic!("unrecognized option for download-ci-llvm: {:?}", other) + } + } + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + assert!( + self.rust_info.is_managed_git_subrepository(), + "Can't run `Config::last_modified_commit` on a non-git source." + ); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = helpers::git(Some(&self.src)); + git.args(["diff-index", "--quiet", &commit, "--"]).args(modified_paths); + + let has_changes = !t!(git.as_command_mut().status()).success(); + if has_changes { + if if_unchanged { + if self.is_verbose() { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_part6.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part6.rs new file mode 100644 index 00000000..038a7046 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part6.rs @@ -0,0 +1,63 @@ +pub struct OptimizeVisitor; + +impl serde::de::Visitor<'_> for OptimizeVisitor { + type Value = RustOptimize; + +fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + +fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + if matches!(value, "s" | "z") { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + +fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + +fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) + } +} + +pub fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + +impl RustOptimize { + pub(crate) fn is_release(&self) -> bool { + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } + } + + pub(crate) fn get_opt_level(&self) -> Option { + match &self { + RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), + RustOptimize::Bool(_) => None, + } + } +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_part7.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part7.rs new file mode 100644 index 00000000..fd40910d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/config_part7.rs @@ -0,0 +1,4 @@ + + + + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_toml.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_toml.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_types.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_types.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/config_utils.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/config_utils.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/debug_info_level.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/debug_info_level.rs new file mode 100644 index 00000000..4433c3d6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/debug_info_level.rs @@ -0,0 +1,61 @@ +use crate::prelude::*; +use serde::Deserializer; +use crate::core::config::string_or_int::StringOrInt; + +#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] +pub enum DebuginfoLevel { + #[default] + None, + LineDirectivesOnly, + LineTablesOnly, + Limited, + Full, +} + +// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only +// deserializes i64, and derive() only generates visit_u64 +impl<'de> Deserialize<'de> for DebuginfoLevel { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + + + Ok(match Deserialize::deserialize(deserializer)? { + StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, + StringOrInt::Int(0) => DebuginfoLevel::None, + StringOrInt::String(s) if s == "line-directives-only" => { + DebuginfoLevel::LineDirectivesOnly + } + StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, + StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, + StringOrInt::Int(1) => DebuginfoLevel::Limited, + StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, + StringOrInt::Int(2) => DebuginfoLevel::Full, + StringOrInt::Int(n) => { + let other = serde::de::Unexpected::Signed(n); + return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); + } + StringOrInt::String(s) => { + let other = serde::de::Unexpected::Str(&s); + return Err(D::Error::invalid_value( + other, + &"expected none, line-tables-only, limited, or full", + )); + } + }) + } +} +/// Suitable for passing to `-C debuginfo` +impl Display for DebuginfoLevel { +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DebuginfoLevel::*; + f.write_str(match self { + None => "0", + LineDirectivesOnly => "line-directives-only", + LineTablesOnly => "line-tables-only", + Limited => "1", + Full => "2", + }) + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/dist.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/dist.rs new file mode 100644 index 00000000..85c375af --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/dist.rs @@ -0,0 +1,13 @@ +use config_macros::define_config; + +define_config! { + struct Dist { + sign_folder: Option = "sign-folder", + upload_addr: Option = "upload-addr", + src_tarball: Option = "src-tarball", + compression_formats: Option> = "compression-formats", + compression_profile: Option = "compression-profile", + include_mingw_linker: Option = "include-mingw-linker", + vendor: Option = "vendor", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/dry_run.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/dry_run.rs new file mode 100644 index 00000000..75c550d3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/dry_run.rs @@ -0,0 +1,18 @@ +/// This file is embedded in the overlay directory of the tarball sources. It is +/// useful in scenarios where developers want to see how the tarball sources were +/// generated. +/// +/// We also use this file to compare the host's config.toml against the CI rustc builder +/// configuration to detect any incompatible options. +pub const BUILDER_CONFIG_FILENAME: &str = "builder-config"; + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/flags.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/flags.rs new file mode 100644 index 00000000..87f58006 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/flags.rs @@ -0,0 +1,221 @@ +//! Command-line interface of the bootstrap build system. +//! +//! This module implements the command-line parsing of the build system which +//! has various flags to configure how it's run. + +use std::path::{Path, PathBuf}; + +use clap::{CommandFactory, Parser, ValueEnum}; + +use crate::core::build_steps::setup::Profile; +use crate::core::builder::{Builder, Kind}; +use crate::core::config::{Config, TargetSelectionList, target_selection_list}; +use crate::{Build, DocTests}; +pub use crate::core::config::subcommand::get_completion; + + + +pub enum Warnings { + Default, + Deny, + Warn, +} + +pub enum Color { + Auto, + Always, + Never, +} + +#[derive(Debug, Parser)] +#[command( + override_usage = "x.py [options] [...]", + disable_help_subcommand(true), + about = "", + next_line_help(false) +)] +pub struct Flags { + #[command(subcommand)] + pub cmd: Subcommand, + + #[arg(global = true, short, long, action = clap::ArgAction::Count)] + /// use verbose output (-vv for very verbose) + pub verbose: u8, // each extra -v after the first is passed to Cargo + #[arg(global = true, short, long)] + /// use incremental compilation + pub incremental: bool, + #[arg(global = true, long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] + /// TOML configuration file for build + pub config: Option, + #[arg(global = true, long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// Build directory, overrides `build.build-dir` in `config.toml` + pub build_dir: Option, + + #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] + /// build target of the stage0 compiler + pub build: Option, + + #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] + /// host targets to build + pub host: Option, + + #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] + /// target targets to build + pub target: Option, + + #[arg(global = true, long, value_name = "PATH")] + /// build paths to exclude + pub exclude: Vec, // keeping for client backward compatibility + #[arg(global = true, long, value_name = "PATH")] + /// build paths to skip + pub skip: Vec, + #[arg(global = true, long)] + /// include default paths in addition to the provided ones + pub include_default_paths: bool, + + #[arg(global = true, value_hint = clap::ValueHint::Other, long)] + pub rustc_error_format: Option, + + #[arg(global = true, long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] + /// command to run on failure + pub on_fail: Option, + #[arg(global = true, long)] + /// dry run; don't build anything + pub dry_run: bool, + /// Indicates whether to dump the work done from bootstrap shims + #[arg(global = true, long)] + pub dump_bootstrap_shims: bool, + #[arg(global = true, value_hint = clap::ValueHint::Other, value_name = "N")] + /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the + /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) + pub stage: Option, + + #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage: Vec, + #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) of the standard library to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage_std: Vec, + #[arg(global = true, long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// path to the root of the rust checkout + pub src: Option, + + #[arg( + global = true, + short, + long, + value_hint = clap::ValueHint::Other, + value_name = "JOBS" + )] + /// number of jobs to run in parallel + pub jobs: Option, + // This overrides the deny-warnings configuration option, + // which passes -Dwarnings to the compiler invocations. + #[arg(global = true, long)] + #[arg(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] + /// if value is deny, will deny warnings + /// if value is warn, will emit warnings + /// otherwise, use the default configured behaviour + pub warnings: Warnings, + + #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] + /// rustc error format + pub error_format: Option, + #[arg(global = true, long)] + /// use message-format=json + pub json_output: bool, + + #[arg(global = true, long, value_name = "STYLE")] + #[arg(value_enum, default_value_t = Color::Auto)] + /// whether to use color in cargo and rustc output + pub color: Color, + + #[arg(global = true, long)] + /// Bootstrap uses this value to decide whether it should bypass locking the build process. + /// This is rarely needed (e.g., compiling the std library for different targets in parallel). + /// + /// Unless you know exactly what you are doing, you probably don't need this. + pub bypass_bootstrap_lock: bool, + + /// generate PGO profile with rustc build + #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_generate: Option, + /// use PGO profile for rustc build + #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_use: Option, + /// use PGO profile for LLVM build + #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub llvm_profile_use: Option, + // LLVM doesn't support a custom location for generating profile + // information. + // + // llvm_out/build/profiles/ is the location this writes to. + /// generate PGO profile with llvm built for rustc + #[arg(global = true, long)] + pub llvm_profile_generate: bool, + /// Enable BOLT link flags + #[arg(global = true, long)] + pub enable_bolt_settings: bool, + /// Skip stage0 compiler validation + #[arg(global = true, long)] + pub skip_stage0_validation: bool, + /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. + #[arg(global = true, long)] + pub reproducible_artifact: Vec, + #[arg(global = true)] + /// paths for the subcommand + pub paths: Vec, + /// override options in config.toml + #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] + pub set: Vec, + /// arguments passed to subcommands + #[arg(global = true, last(true), value_name = "ARGS")] + pub free_args: Vec, +} + +impl Flags { + /// Check if ` -h -v` was passed. + /// If yes, print the available paths and return `true`. + pub fn try_parse_verbose_help(args: &[String]) -> bool { + // We need to check for ` -h -v`, in which case we list the paths + #[derive(Parser)] + #[command(disable_help_flag(true))] + struct HelpVerboseOnly { + #[arg(short, long)] + help: bool, + #[arg(global = true, short, long, action = clap::ArgAction::Count)] + pub verbose: u8, + #[arg(value_enum)] + cmd: Kind, + } + if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = + HelpVerboseOnly::try_parse_from(normalize_args(args)) + { + println!("NOTE: updating submodules before printing available paths"); + let config = Config::parse(Self::parse(&[String::from("build")])); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + if let Some(s) = paths { + println!("{s}"); + } else { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } + true + } else { + false + } + } + + pub fn parse(args: &[String]) -> Self { + Flags::parse_from(normalize_args(args)) + } +} + +pub fn normalize_args(args: &[String]) -> Vec { + let first = String::from("x.py"); + let it = std::iter::once(first).chain(args.iter().cloned()); + it.collect() +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/install.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/install.rs new file mode 100644 index 00000000..14c9cbd7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/install.rs @@ -0,0 +1,14 @@ +use config_macros::define_config; + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option = "prefix", + sysconfdir: Option = "sysconfdir", + docdir: Option = "docdir", + bindir: Option = "bindir", + libdir: Option = "libdir", + mandir: Option = "mandir", + datadir: Option = "datadir", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/lib.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/lib.rs new file mode 100644 index 00000000..cb8ea9c2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/lib.rs @@ -0,0 +1,83 @@ +use crate::prelude::*; + +pub mod build; +pub mod changeid; +pub mod ci; +pub mod ciconfig; +pub mod color; +pub mod config_base; +pub mod config_ci; +pub mod config_part2; +pub mod config_part3; +pub mod config_part4; + +pub mod config_part6; +pub mod config_part7; +pub mod config_toml; +pub mod config_types; +pub mod config_utils; +pub mod debug_info_level; +pub mod dist; +pub mod dry_run; +pub mod flags; +pub mod install; +pub mod lld_mode; +pub mod llvm; +pub mod llvm_lib_unwind; +pub mod merge; +pub mod replaceop; +pub mod rust; +pub mod rust_optimize; +pub mod rustclto; +pub mod rustfmt; +pub mod splitdebuginfo; +pub mod string_or_int; +pub mod stringorbool; +pub mod subcommand; +pub use crate::core::config::subcommand::Subcommand; +pub mod target_selection; +pub mod target; +pub mod tomlconfig; +pub mod tomltarget; +pub mod warnings; + + +pub use target_selection::*; +pub use build::*; +pub use changeid::*; +pub use ci::*; +pub use ciconfig::*; +pub use color::*; +pub use config_base::*; +pub use config_ci::*; +pub use config_part2::*; +pub use config_part3::*; +pub use config_part4::*; + +pub use config_part6::*; +pub use config_part7::*; +pub use config_toml::*; +pub use config_types::*; +pub use config_utils::*; +pub use debug_info_level::*; +pub use dist::*; +pub use dry_run::*; +pub use flags::*; +pub use install::*; +pub use llvm::*; +pub use llvm_lib_unwind::*; +pub use merge::*; +pub use replaceop::*; +pub use rust::*; +pub use rust_optimize::*; +pub use rustclto::*; +pub use rustfmt::*; +pub use splitdebuginfo::*; +pub use string_or_int::*; +pub use stringorbool::*; +pub use subcommand::*; +pub use target_selection::*; +pub use target::*; +pub use tomlconfig::*; +pub use tomltarget::*; +pub use warnings::*; diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/lld_mode.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/lld_mode.rs new file mode 100644 index 00000000..13f7aada --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/lld_mode.rs @@ -0,0 +1,76 @@ +use crate::prelude::*; +use serde::Deserializer; + +/// LLD in bootstrap works like this: +/// - Self-contained lld: use `rust-lld` from the compiler's sysroot +/// - External: use an external `lld` binary +/// +/// It is configured depending on the target: +/// 1) Everything except MSVC +/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` +/// - External: `-Clinker-flavor=gnu-lld-cc` +/// 2) MSVC +/// - Self-contained: `-Clinker=` +/// - External: `-Clinker=lld` +use crate::prelude::*; +use serde::de::Error; + +#[derive(Copy, Clone, Default, Debug, PartialEq)] +pub enum LldMode { + /// Do not use LLD + #[default] + Unused, + /// Use `rust-lld` from the compiler's sysroot + SelfContained, + /// Use an externally provided `lld` binary. + /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has + /// to be in $PATH. + External, +} + +impl LldMode { + pub fn is_used(&self) -> bool { + match self { + LldMode::SelfContained | LldMode::External => true, + LldMode::Unused => false, + } + } +} + + +impl<'de> Deserialize<'de> for LldMode { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct LldModeVisitor; + + impl serde::de::Visitor<'_> for LldModeVisitor { + type Value = LldMode; + +fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("one of true, 'self-contained' or 'external'") + } + +fn visit_bool(self, v: bool) -> Result + where + E: serde::de::Error, + { + Ok(if v { LldMode::External } else { LldMode::Unused }) + } + +fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + match v { + "external" => Ok(LldMode::External), + "self-contained" => Ok(LldMode::SelfContained), + _ => Err(E::custom(&format!("unknown mode {}", v))), + } + } + } + + deserializer.deserialize_any(LldModeVisitor) + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/llvm.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/llvm.rs new file mode 100644 index 00000000..872d46b7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/llvm.rs @@ -0,0 +1,39 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + optimize: Option = "optimize", + thin_lto: Option = "thin-lto", + release_debuginfo: Option = "release-debuginfo", + assertions: Option = "assertions", + tests: Option = "tests", + enzyme: Option = "enzyme", + plugins: Option = "plugins", + ccache: Option = "ccache", + static_libstdcpp: Option = "static-libstdcpp", + libzstd: Option = "libzstd", + ninja: Option = "ninja", + targets: Option = "targets", + experimental_targets: Option = "experimental-targets", + link_jobs: Option = "link-jobs", + link_shared: Option = "link-shared", + version_suffix: Option = "version-suffix", + clang_cl: Option = "clang-cl", + cflags: Option = "cflags", + cxxflags: Option = "cxxflags", + ldflags: Option = "ldflags", + use_libcxx: Option = "use-libcxx", + use_linker: Option = "use-linker", + allow_old_toolchain: Option = "allow-old-toolchain", + offload: Option = "offload", + polly: Option = "polly", + clang: Option = "clang", + enable_warnings: Option = "enable-warnings", + download_ci_llvm: Option = "download-ci-llvm", + build_config: Option> = "build-config", + enable_projects: Option = "enable-projects", + } +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/llvm_lib_unwind.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/llvm_lib_unwind.rs new file mode 100644 index 00000000..39d403ee --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/llvm_lib_unwind.rs @@ -0,0 +1,21 @@ +use crate::prelude::*; +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum LlvmLibunwind { + #[default] + No, + InTree, + System, +} + +impl FromStr for LlvmLibunwind { + type Err = String; + +fn from_str(value: &str) -> Result { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/merge.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/merge.rs new file mode 100644 index 00000000..e69de29b diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/replaceop.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/replaceop.rs new file mode 100644 index 00000000..3b34b9ec --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/replaceop.rs @@ -0,0 +1,10 @@ +/// Describes how to handle conflicts in merging two [`TomlConfig`] +#[derive(Copy, Clone, Debug)] +enum ReplaceOpt { + /// Silently ignore a duplicated value + IgnoreDuplicate, + /// Override the current value, even if it's `Some` + Override, + /// Exit with an error on duplicate values + ErrorOnDuplicate, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/rust.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/rust.rs new file mode 100644 index 00000000..64ca764b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/rust.rs @@ -0,0 +1,63 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option = "optimize", + debug: Option = "debug", + codegen_units: Option = "codegen-units", + codegen_units_std: Option = "codegen-units-std", + rustc_debug_assertions: Option = "debug-assertions", + randomize_layout: Option = "randomize-layout", + std_debug_assertions: Option = "debug-assertions-std", + overflow_checks: Option = "overflow-checks", + overflow_checks_std: Option = "overflow-checks-std", + debug_logging: Option = "debug-logging", + debuginfo_level: Option = "debuginfo-level", + debuginfo_level_rustc: Option = "debuginfo-level-rustc", + debuginfo_level_std: Option = "debuginfo-level-std", + debuginfo_level_tools: Option = "debuginfo-level-tools", + debuginfo_level_tests: Option = "debuginfo-level-tests", + backtrace: Option = "backtrace", + incremental: Option = "incremental", + parallel_compiler: Option = "parallel-compiler", + default_linker: Option = "default-linker", + channel: Option = "channel", + description: Option = "description", + musl_root: Option = "musl-root", + rpath: Option = "rpath", + strip: Option = "strip", + frame_pointers: Option = "frame-pointers", + stack_protector: Option = "stack-protector", + verbose_tests: Option = "verbose-tests", + optimize_tests: Option = "optimize-tests", + codegen_tests: Option = "codegen-tests", + omit_git_hash: Option = "omit-git-hash", + dist_src: Option = "dist-src", + save_toolstates: Option = "save-toolstates", + codegen_backends: Option> = "codegen-backends", + llvm_bitcode_linker: Option = "llvm-bitcode-linker", + lld: Option = "lld", + lld_mode: Option = "use-lld", + llvm_tools: Option = "llvm-tools", + deny_warnings: Option = "deny-warnings", + backtrace_on_ice: Option = "backtrace-on-ice", + verify_llvm_ir: Option = "verify-llvm-ir", + thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", + remap_debuginfo: Option = "remap-debuginfo", + jemalloc: Option = "jemalloc", + test_compare_mode: Option = "test-compare-mode", + llvm_libunwind: Option = "llvm-libunwind", + control_flow_guard: Option = "control-flow-guard", + ehcont_guard: Option = "ehcont-guard", + new_symbol_mangling: Option = "new-symbol-mangling", + profile_generate: Option = "profile-generate", + profile_use: Option = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option = "download-rustc", + lto: Option = "lto", + validate_mir_opts: Option = "validate-mir-opts", + std_features: Option> = "std-features", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/rust_optimize.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/rust_optimize.rs new file mode 100644 index 00000000..36adbee9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/rust_optimize.rs @@ -0,0 +1,25 @@ +use crate::prelude::*; +use serde::Deserializer; +use crate::core::config::config_part6::OptimizeVisitor; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RustOptimize { + String(String), + Int(u8), + Bool(bool), +} + +impl Default for RustOptimize { +fn default() -> RustOptimize { + RustOptimize::Bool(false) + } +} + +impl<'de> Deserialize<'de> for RustOptimize { +fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/rustclto.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/rustclto.rs new file mode 100644 index 00000000..993e25d6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/rustclto.rs @@ -0,0 +1,24 @@ + +/// LTO mode used for compiling rustc itself. +#[derive(Default, Clone, PartialEq, Debug)] +pub enum RustcLto { + Off, + #[default] + ThinLocal, + Thin, + Fat, +} + +impl std::str::FromStr for RustcLto { + type Err = String; + +fn from_str(s: &str) -> Result { + match s { + "thin-local" => Ok(RustcLto::ThinLocal), + "thin" => Ok(RustcLto::Thin), + "fat" => Ok(RustcLto::Fat), + "off" => Ok(RustcLto::Off), + _ => Err(format!("Invalid value for rustc LTO: {s}")), + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/rustfmt.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/rustfmt.rs new file mode 100644 index 00000000..f346f645 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/rustfmt.rs @@ -0,0 +1,9 @@ +use crate::prelude::*; +#[derive(Clone, Debug, Default)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + #[default] + LazyEvaluated, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/splitdebuginfo.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/splitdebuginfo.rs new file mode 100644 index 00000000..c0f3b937 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/splitdebuginfo.rs @@ -0,0 +1,35 @@ +use crate::prelude::*; +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + #[default] + Off, +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + +fn from_str(s: &str) -> Result { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.example.toml`. +pub fn default_for_platform(target: TargetSelection) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.is_windows() { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/string_or_int.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/string_or_int.rs new file mode 100644 index 00000000..7de6d383 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/string_or_int.rs @@ -0,0 +1,8 @@ +use crate::prelude::*; + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum StringOrInt { + String(String), + Int(i64), +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/stringorbool.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/stringorbool.rs new file mode 100644 index 00000000..b9b65a22 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/stringorbool.rs @@ -0,0 +1,19 @@ +use crate::prelude::*; +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { +fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +impl StringOrBool { +pub fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand.rs new file mode 100644 index 00000000..3592d55e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand.rs @@ -0,0 +1,181 @@ +use crate::prelude::*; + +pub mod subcommand_groups; +use subcommand_groups::{QaTool, BuildTool, DistTool, MiscTool}; + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum Subcommand { + Qa(QaTool), + Build(BuildTool), + Dist(DistTool), + Misc(MiscTool), +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Qa(qa_tool) => match qa_tool { + QaTool::Bench { .. } => Kind::Bench, + QaTool::Check { .. } => Kind::Check, + QaTool::Clippy { .. } => Kind::Clippy, + QaTool::Fix { .. } => Kind::Fix, + QaTool::Format { .. } => Kind::Format, + QaTool::Test { .. } => Kind::Test, + QaTool::Miri { .. } => Kind::Miri, + QaTool::Suggest { .. } => Kind::Suggest, + QaTool::Perf { .. } => Kind::Perf, + }, + Subcommand::Build(build_tool) => match build_tool { + BuildTool::Build { .. } => Kind::Build, + BuildTool::Doc { .. } => Kind::Doc, + }, + Subcommand::Dist(dist_tool) => match dist_tool { + DistTool::Dist { .. } => Kind::Dist, + DistTool::Install { .. } => Kind::Install, + }, + Subcommand::Misc(misc_tool) => match misc_tool { + MiscTool::Clean { .. } => Kind::Clean, + MiscTool::Run { .. } => Kind::Run, + MiscTool::Setup { .. } => Kind::Setup, + MiscTool::Vendor { .. } => Kind::Vendor, + }, + } + } + + pub fn compiletest_rustc_args(&self) -> Vec<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref compiletest_rustc_args, .. }) => { + compiletest_rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + } + } + + pub fn fail_fast(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { no_fail_fast, .. }) | Subcommand::Qa(QaTool::Miri { no_fail_fast, .. }) => { + !no_fail_fast + } + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match self { + Subcommand::Qa(QaTool::Test { doc, no_doc, .. }) | Subcommand::Qa(QaTool::Miri { no_doc, doc, .. }) => { + if *doc { + DocTests::Only + } else if *no_doc { + DocTests::No + } else { + DocTests::Yes + } + } + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { bless, .. }) => *bless, + _ => false, + } + } + + pub fn extra_checks(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref extra_checks, .. }) => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + + pub fn only_modified(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { only_modified, .. }) => *only_modified, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { force_rerun, .. }) => *force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match self { + Subcommand::Qa(QaTool::Test { rustfix_coverage, .. }) => *rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref compare_mode, .. }) => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref pass, .. }) => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match self { + Subcommand::Qa(QaTool::Test { ref run, .. }) => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match self { + Subcommand::Build(BuildTool::Doc { open, .. }) => *open, + _ => false, + } + } + + pub fn json(&self) -> bool { + match self { + Subcommand::Build(BuildTool::Doc { json, .. }) => *json, + _ => false, + } + } + + pub fn vendor_versioned_dirs(&self) -> bool { + match self { + Subcommand::Misc(MiscTool::Vendor { versioned_dirs, .. }) => *versioned_dirs, + _ => false, + } + } + + pub fn vendor_sync_args(&self) -> Vec { + match self { + Subcommand::Misc(MiscTool::Vendor { sync, .. }) => sync.clone(), + _ => vec![], + } + } +} + +/// Returns the shell completion for a given shell, if the result differs from the current +/// content of `path`. If `path` does not exist, always returns `Some`. +pub fn get_completion(shell: G, path: &Path) -> Option { + let mut cmd = Flags::command(); + let current = if !path.exists() { + String::new() + } else { + std::fs::read_to_string(path).unwrap_or_else(|_| { + eprintln!("couldn't read {}", path.display()); + crate::exit!(1) + }) + }; + let mut buf = Vec::new(); + clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); + if buf == current.as_bytes() { + return None; + } + Some(String::from_utf8(buf).expect("completion script should be UTF-8")) +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand_groups.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand_groups.rs new file mode 100644 index 00000000..5226d214 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/subcommand_groups.rs @@ -0,0 +1,112 @@ +use crate::prelude::*; + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum QaTool { + Check { + #[arg(long)] + all_targets: bool, + }, + Clippy { + #[arg(long)] + fix: bool, + #[arg(long, requires = "fix")] + allow_dirty: bool, + #[arg(long, requires = "fix")] + allow_staged: bool, + #[arg(global = true, short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] + allow: Vec, + #[arg(global = true, short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] + deny: Vec, + #[arg(global = true, short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] + warn: Vec, + #[arg(global = true, short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] + forbid: Vec, + }, + Fix, + Format { + #[arg(long)] + check: bool, + #[arg(long)] + all: bool, + }, + Test { + #[arg(long)] + no_fail_fast: bool, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + test_args: Vec, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + compiletest_rustc_args: Vec, + #[arg(long)] + no_doc: bool, + #[arg(long)] + doc: bool, + #[arg(long)] + bless: bool, + #[arg(long, value_name = "check | build | run")] + pass: Option, + #[arg(long, value_name = "auto | always | never")] + run: Option, + #[arg(long)] + rustfix_coverage: bool, + }, + Miri { + #[arg(long)] + no_fail_fast: bool, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + test_args: Vec, + #[arg(long)] + no_doc: bool, + #[arg(long)] + doc: bool, + }, + Bench { + #[arg(long, allow_hyphen_values(true))] + test_args: Vec, + }, + Suggest { + #[arg(long)] + run: bool, + }, + Perf, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum BuildTool { + Build, + Doc { + #[arg(long)] + open: bool, + #[arg(long)] + json: bool, + }, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum DistTool { + Dist, + Install, +} + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum MiscTool { + Clean { + #[arg(long)] + all: bool, + #[arg(long, value_name = "N")] + stage: Option, + }, + Run { + #[arg(long, allow_hyphen_values(true))] + args: Vec, + }, + Setup { + #[arg(value_name = "|hook|editor|link")] + profile: Option, + }, + Vendor { + #[arg(long)] + sync: Vec, + #[arg(long)] + versioned_dirs: bool, + }, +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/target.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/target.rs new file mode 100644 index 00000000..a4a9ffba --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/target.rs @@ -0,0 +1,13 @@ +use crate::prelude::*; +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { + target.no_std = true; + } + if triple.contains("emscripten") { + target.runner = Some("node".into()); + } + target + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/target_selection.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/target_selection.rs new file mode 100644 index 00000000..d8f11b5b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/target_selection.rs @@ -0,0 +1,148 @@ +use crate::prelude::*; + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. +// Making !Copy is highly nontrivial! +pub struct TargetSelection { + pub triple: Interned, + file: Option>, + synthetic: bool, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file, synthetic: false } + } + + pub fn create_synthetic(triple: &str, file: &str) -> Self { + Self { + triple: INTERNER.intern_str(triple), + file: Some(INTERNER.intern_str(file)), + synthetic: true, + } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } + + // See src/bootstrap/synthetic_targets.rs + pub fn is_synthetic(&self) -> bool { + self.synthetic + } + + pub fn is_msvc(&self) -> bool { + self.contains("msvc") + } + + pub fn is_windows(&self) -> bool { + self.contains("windows") + } + + pub fn is_windows_gnu(&self) -> bool { + self.ends_with("windows-gnu") + } + + /// Path to the file defining the custom target, if any. + pub fn filepath(&self) -> Option<&Path> { + self.file.as_ref().map(Path::new) + } +} + +impl fmt::Display for TargetSelection { +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({file})")?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { +pub fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl PartialEq<&str> for TargetSelection { +fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +// Targets are often used as directory names throughout bootstrap. +// This impl makes it more ergonomics to use them as such. +impl AsRef for TargetSelection { +fn as_ref(&self) -> &Path { + self.triple.as_ref() + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option, + pub llvm_has_rust_patches: Option, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option, + pub llvm_libunwind: Option, + pub cc: Option, + pub cxx: Option, + pub ar: Option, + pub ranlib: Option, + pub default_linker: Option, + pub linker: Option, + pub split_debuginfo: Option, + pub sanitizers: Option, + pub profiler: Option, + pub rpath: Option, + pub crt_static: Option, + pub musl_root: Option, + pub musl_libdir: Option, + pub wasi_root: Option, + pub qemu_rootfs: Option, + pub runner: Option, + pub no_std: bool, + pub codegen_backends: Option>, +} diff --git a/standalonex/src/src/core/config/tests.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/tests.rs similarity index 94% rename from standalonex/src/src/core/config/tests.rs rename to standalonex/src/bootstrap/src/core/config_standalone/src/tests.rs index 73d78724..00cd99ce 100644 --- a/standalonex/src/src/core/config/tests.rs +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/tests.rs @@ -22,7 +22,7 @@ pub(crate) fn parse(config: &str) -> Config { #[test] #[ignore] -fn download_ci_llvm() { +pub fn download_ci_llvm() { let config = parse(""); let is_available = llvm::is_ci_llvm_available(&config, config.llvm_assertions); if is_available { @@ -56,8 +56,8 @@ fn download_ci_llvm() { // - https://github.com/rust-lang/rust/issues/109120 // - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 #[test] -fn detect_src_and_out() { - fn test(cfg: Config, build_dir: Option<&str>) { +pub fn detect_src_and_out() { +pub fn test(cfg: Config, build_dir: Option<&str>) { // This will bring absolute form of `src/bootstrap` path let current_dir = std::env::current_dir().unwrap(); @@ -108,12 +108,12 @@ fn detect_src_and_out() { } #[test] -fn clap_verify() { +pub fn clap_verify() { Flags::command().debug_assert(); } #[test] -fn override_toml() { +pub fn override_toml() { let config = Config::parse_inner( Flags::parse(&[ "check".to_owned(), @@ -208,7 +208,7 @@ runner = "x86_64-runner" #[test] #[should_panic] -fn override_toml_duplicate() { +pub fn override_toml_duplicate() { Config::parse_inner( Flags::parse(&[ "check".to_owned(), @@ -221,8 +221,8 @@ fn override_toml_duplicate() { } #[test] -fn profile_user_dist() { - fn get_toml(file: &Path) -> Result { +pub fn profile_user_dist() { +pub fn get_toml(file: &Path) -> Result { let contents = if file.ends_with("config.toml") || env::var_os("RUST_BOOTSTRAP_CONFIG").is_some() { "profile = \"user\"".to_owned() @@ -237,7 +237,7 @@ fn profile_user_dist() { } #[test] -fn rust_optimize() { +pub fn rust_optimize() { assert!(parse("").rust_optimize.is_release()); assert!(!parse("rust.optimize = false").rust_optimize.is_release()); assert!(parse("rust.optimize = true").rust_optimize.is_release()); @@ -250,12 +250,12 @@ fn rust_optimize() { #[test] #[should_panic] -fn invalid_rust_optimize() { +pub fn invalid_rust_optimize() { parse("rust.optimize = \"a\""); } #[test] -fn verify_file_integrity() { +pub fn verify_file_integrity() { let config = parse(""); let tempfile = config.tempdir().join(".tmp-test-file"); @@ -271,7 +271,7 @@ fn verify_file_integrity() { } #[test] -fn rust_lld() { +pub fn rust_lld() { assert!(matches!(parse("").lld_mode, LldMode::Unused)); assert!(matches!(parse("rust.use-lld = \"self-contained\"").lld_mode, LldMode::SelfContained)); assert!(matches!(parse("rust.use-lld = \"external\"").lld_mode, LldMode::External)); @@ -281,12 +281,12 @@ fn rust_lld() { #[test] #[should_panic] -fn parse_config_with_unknown_field() { +pub fn parse_config_with_unknown_field() { parse("unknown-key = 1"); } #[test] -fn parse_change_id_with_unknown_field() { +pub fn parse_change_id_with_unknown_field() { let config = r#" change-id = 3461 unknown-key = 1 @@ -297,7 +297,7 @@ fn parse_change_id_with_unknown_field() { } #[test] -fn order_of_clippy_rules() { +pub fn order_of_clippy_rules() { let args = vec![ "clippy".to_string(), "--fix".to_string(), @@ -329,7 +329,7 @@ fn order_of_clippy_rules() { } #[test] -fn clippy_rule_separate_prefix() { +pub fn clippy_rule_separate_prefix() { let args = vec!["clippy".to_string(), "-A clippy:all".to_string(), "-W clippy::style".to_string()]; let config = Config::parse(Flags::parse(&args)); @@ -347,7 +347,7 @@ fn clippy_rule_separate_prefix() { } #[test] -fn verbose_tests_default_value() { +pub fn verbose_tests_default_value() { let config = Config::parse(Flags::parse(&["build".into(), "compiler".into()])); assert_eq!(config.verbose_tests, false); @@ -356,7 +356,7 @@ fn verbose_tests_default_value() { } #[test] -fn parse_rust_std_features() { +pub fn parse_rust_std_features() { let config = parse("rust.std-features = [\"panic-unwind\", \"backtrace\"]"); let expected_features: BTreeSet = ["panic-unwind", "backtrace"].into_iter().map(|s| s.to_string()).collect(); @@ -364,7 +364,7 @@ fn parse_rust_std_features() { } #[test] -fn parse_rust_std_features_empty() { +pub fn parse_rust_std_features_empty() { let config = parse("rust.std-features = []"); let expected_features: BTreeSet = BTreeSet::new(); assert_eq!(config.rust_std_features, expected_features); @@ -372,17 +372,17 @@ fn parse_rust_std_features_empty() { #[test] #[should_panic] -fn parse_rust_std_features_invalid() { +pub fn parse_rust_std_features_invalid() { parse("rust.std-features = \"backtrace\""); } #[test] -fn parse_jobs() { +pub fn parse_jobs() { assert_eq!(parse("build.jobs = 1").jobs, Some(1)); } #[test] -fn jobs_precedence() { +pub fn jobs_precedence() { // `--jobs` should take precedence over using `--set build.jobs`. let config = Config::parse_inner( @@ -435,7 +435,7 @@ fn jobs_precedence() { } #[test] -fn check_rustc_if_unchanged_paths() { +pub fn check_rustc_if_unchanged_paths() { let config = parse(""); let normalised_allowed_paths: Vec<_> = RUSTC_IF_UNCHANGED_ALLOWED_PATHS .iter() diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/tomlconfig.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/tomlconfig.rs new file mode 100644 index 00000000..2bcc70f8 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/tomlconfig.rs @@ -0,0 +1,22 @@ +use crate::prelude::*; +/// Structure of the `config.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct TomlConfig { + #[serde(flatten)] + change_id: ChangeIdWrapper, + build: Option, + install: Option, + llvm: Option, + rust: Option, + target: Option>, + dist: Option, + ci: Option, + profile: Option, + stage0_path: Option, +} + diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/tomltarget.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/tomltarget.rs new file mode 100644 index 00000000..7ff46ddd --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/tomltarget.rs @@ -0,0 +1,30 @@ +use crate::prelude::*; +use config_macros::define_config; + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option = "cc", + cxx: Option = "cxx", + ar: Option = "ar", + ranlib: Option = "ranlib", + default_linker: Option = "default-linker", + linker: Option = "linker", + split_debuginfo: Option = "split-debuginfo", + llvm_config: Option = "llvm-config", + llvm_has_rust_patches: Option = "llvm-has-rust-patches", + llvm_filecheck: Option = "llvm-filecheck", + llvm_libunwind: Option = "llvm-libunwind", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + rpath: Option = "rpath", + crt_static: Option = "crt-static", + musl_root: Option = "musl-root", + musl_libdir: Option = "musl-libdir", + wasi_root: Option = "wasi-root", + qemu_rootfs: Option = "qemu-rootfs", + no_std: Option = "no-std", + codegen_backends: Option> = "codegen-backends", + runner: Option = "runner", + } +} diff --git a/standalonex/src/bootstrap/src/core/config_standalone/src/warnings.rs b/standalonex/src/bootstrap/src/core/config_standalone/src/warnings.rs new file mode 100644 index 00000000..966ee7b4 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_standalone/src/warnings.rs @@ -0,0 +1,9 @@ +use crate::prelude::*; +/// Whether to deny warnings, emit them as warnings, or use the default behavior +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Warnings { + Deny, + Warn, + #[default] + Default, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/Cargo.toml b/standalonex/src/bootstrap/src/core/config_utils/Cargo.toml new file mode 100644 index 00000000..aefda735 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "bootstrap-config-utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +toml = "0.5" +serde = "1.0" +serde_derive = "1.0" +stage0_parser_crate = { path = "../../../../stage0_parser_crate" } + + diff --git a/standalonex/src/bootstrap/src/core/config_utils/flake.nix b/standalonex/src/bootstrap/src/core/config_utils/flake.nix new file mode 100644 index 00000000..7acca5db --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/flake.nix @@ -0,0 +1,26 @@ +{ + description = "A minimal development shell for bootstrap-config-utils"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + flake-utils.url = "github:meta-introspector/flake-utils?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; + [ + cargo + rustc + rust-analyzer + clippy + rustfmt + ]; + }; + }); +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/report.sh b/standalonex/src/bootstrap/src/core/config_utils/report.sh new file mode 100644 index 00000000..51ab23af --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/report.sh @@ -0,0 +1,5 @@ + +cd $(dirname "$0") +cargo build > report.txt 2>&1 +cat report.txt | sort | uniq -c | sort -n > report2.txt + diff --git a/standalonex/src/bootstrap/src/core/config_utils/report.txt b/standalonex/src/bootstrap/src/core/config_utils/report.txt new file mode 100644 index 00000000..47988867 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/report.txt @@ -0,0 +1,528 @@ +warning: virtual workspace defaulting to `resolver = "1"` despite one or more workspace members being on edition 2021 which implies `resolver = "2"` +note: to keep the current resolver, specify `workspace.resolver = "1"` in the workspace root's manifest +note: to use the edition 2021 resolver, specify `workspace.resolver = "2"` in the workspace root's manifest +note: for more details see https://doc.rust-lang.org/cargo/reference/resolver.html#resolver-versions + Compiling bootstrap-config-utils v0.1.0 (/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix/standalonex/src/bootstrap/src/core/config_utils) +error[E0433]: failed to resolve: could not find `DryRun` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:15:16 + | +15 | crate::DryRun::Disabled => false, + | ^^^^^^ could not find `DryRun` in the crate root + +error[E0433]: failed to resolve: could not find `DryRun` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:16:16 + | +16 | crate::DryRun::SelfCheck | crate::DryRun::UserSelected => true, + | ^^^^^^ could not find `DryRun` in the crate root + +error[E0433]: failed to resolve: could not find `DryRun` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:16:43 + | +16 | crate::DryRun::SelfCheck | crate::DryRun::UserSelected => true, + | ^^^^^^ could not find `DryRun` in the crate root + +error[E0412]: cannot find type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:7:33 + | +7 | pub fn default_opts() -> crate::ParsedConfig { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | +2 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +7 - pub fn default_opts() -> crate::ParsedConfig { +7 + pub fn default_opts() -> ParsedConfig { + | + +error[E0422]: cannot find struct, variant or union type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:14:12 + | +14 | crate::ParsedConfig { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 2 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +14 - crate::ParsedConfig { +14 + ParsedConfig { + | + +error[E0433]: failed to resolve: could not find `TargetSelection` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:42:23 + | +42 | build: crate::TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + | ^^^^^^^^^^^^^^^ could not find `TargetSelection` in the crate root + | +help: consider importing this struct + | + 2 + use crate::target_selection::TargetSelection; + | +help: if you import `TargetSelection`, refer to it directly + | +42 - build: crate::TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), +42 + build: TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + | + +error[E0422]: cannot find struct, variant or union type `LocalCiConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:51:25 + | +51 | ci: Some(crate::LocalCiConfig { + | ^^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 2 + use crate::local_ci_config::LocalCiConfig; + | +help: if you import `LocalCiConfig`, refer to it directly + | +51 - ci: Some(crate::LocalCiConfig { +51 + ci: Some(LocalCiConfig { + | + +error[E0412]: cannot find type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:6:41 + | +6 | pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | +2 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +6 - pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { +6 + pub fn get_builder_toml(config: &ParsedConfig, build_name: &str) -> Result { + | + +error[E0412]: cannot find type `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:6:90 + | +6 | pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + | ^^^^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | +2 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +6 - pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { +6 + pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + | + +error[E0433]: failed to resolve: could not find `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:8:26 + | +8 | return Ok(crate::LocalTomlConfig::default()); + | ^^^^^^^^^^^^^^^ could not find `LocalTomlConfig` in the crate root + | +help: consider importing this struct + | +2 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +8 - return Ok(crate::LocalTomlConfig::default()); +8 + return Ok(LocalTomlConfig::default()); + | + +error[E0412]: cannot find type `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:13:54 + | +13 | pub(crate) fn get_toml(file: &Path) -> Result { + | ^^^^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 1 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +13 - pub(crate) fn get_toml(file: &Path) -> Result { +13 + pub(crate) fn get_toml(file: &Path) -> Result { + | + +error[E0433]: failed to resolve: could not find `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:19:47 + | +19 | .and_then(|table: toml::Value| crate::LocalTomlConfig::deserialize(table)) + | ^^^^^^^^^^^^^^^ could not find `LocalTomlConfig` in the crate root + | +help: consider importing this struct + | + 1 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +19 - .and_then(|table: toml::Value| crate::LocalTomlConfig::deserialize(table)) +19 + .and_then(|table: toml::Value| LocalTomlConfig::deserialize(table)) + | + +error[E0412]: cannot find type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs:1:47 + | +1 | pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | +1 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +1 - pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { +1 + pub fn parse_inner_stage0(config: &mut ParsedConfig, toml: &crate::LocalTomlConfig) { + | + +error[E0412]: cannot find type `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs:1:75 + | +1 | pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { + | ^^^^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | +1 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +1 - pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { +1 + pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &LocalTomlConfig) { + | + +error[E0412]: cannot find type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:46 + | +10 | pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 5 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +10 - pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { +10 + pub fn parse_inner_build(config: &mut ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + | + +error[E0412]: cannot find type `LocalTomlConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:78 + | +10 | pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + | ^^^^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 5 + use crate::local_toml_config::LocalTomlConfig; + | +help: if you import `LocalTomlConfig`, refer to it directly + | +10 - pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { +10 + pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut LocalTomlConfig, flags: &crate::LocalFlags) { + | + +error[E0412]: cannot find type `LocalFlags` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:110 + | +10 | pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + | ^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 5 + use crate::local_flags::LocalFlags; + | +help: if you import `LocalFlags`, refer to it directly + | +10 - pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { +10 + pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &LocalFlags) { + | + +error[E0422]: cannot find struct, variant or union type `LocalBuild` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:11:16 + | +11 | let crate::LocalBuild { + | ^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 5 + use crate::local_build::LocalBuild; + | +help: if you import `LocalBuild`, refer to it directly + | +11 - let crate::LocalBuild { +11 + let LocalBuild { + | + +error[E0412]: cannot find type `ParsedConfig` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:13:32 + | +13 | pub fn dry_run(config: &crate::ParsedConfig) -> bool { + | ^^^^^^^^^^^^ not found in the crate root + | +help: consider importing this struct + | + 1 + use crate::parsed_config::ParsedConfig; + | +help: if you import `ParsedConfig`, refer to it directly + | +13 - pub fn dry_run(config: &crate::ParsedConfig) -> bool { +13 + pub fn dry_run(config: &ParsedConfig) -> bool { + | + +error[E0412]: cannot find type `ParsedConfig` in this scope + --> standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs:2:44 + | +2 | fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig); + | ^^^^^^^^^^^^ not found in this scope + | +help: consider importing this struct + | +1 + use crate::parsed_config::ParsedConfig; + | + +error[E0412]: cannot find type `LocalTomlConfig` in this scope + --> standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs:2:65 + | +2 | fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig); + | ^^^^^^^^^^^^^^^ not found in this scope + | +help: consider importing this struct + | +1 + use crate::local_toml_config::LocalTomlConfig; + | + +error[E0412]: cannot find type `DryRun` in the crate root + --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:42:25 + | +42 | pub dry_run: crate::DryRun, + | ^^^^^^ not found in the crate root + | +help: consider importing this enum + | + 2 + use crate::dry_run::DryRun; + | +help: if you import `DryRun`, refer to it directly + | +42 - pub dry_run: crate::DryRun, +42 + pub dry_run: DryRun, + | + +warning: unused imports: `PathBuf` and `Path` + --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:2:17 + | +2 | use std::path::{PathBuf, Path}; + | ^^^^^^^ ^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default + +warning: unused import: `std::collections::HashMap` + --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:3:5 + | +3 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `serde_derive::Deserialize` + --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:5:5 + | +5 | use serde_derive::Deserialize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `std::path::PathBuf` + --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:2:5 + | +2 | use std::path::PathBuf; + | ^^^^^^^^^^^^^^^^^^ + +warning: unused import: `serde::Deserialize` + --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:3:5 + | +3 | use serde::Deserialize; + | ^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::local_toml_config::LocalTomlConfig` + --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:5:5 + | +5 | use crate::local_toml_config::LocalTomlConfig; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `PathBuf` and `Path` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:3:17 + | +3 | use std::path::{Path, PathBuf}; + | ^^^^ ^^^^^^^ + +warning: unused import: `std::env` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:4:5 + | +4 | use std::env; + | ^^^^^^^^ + +warning: unused import: `toml` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:5:5 + | +5 | use toml; + | ^^^^ + +warning: unused import: `crate::parse_inner_stage0` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:8:5 + | +8 | use crate::parse_inner_stage0; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::get_toml` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:4:5 + | +4 | use crate::get_toml; + | ^^^^^^^^^^^^^^^ + +warning: unused import: `std::fs` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:8:5 + | +8 | use std::fs; + | ^^^^^^^ + +warning: unexpected `cfg` condition value: `bootstrap-self-test` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:25:42 + | +25 | config.config = Some(if cfg!(not(feature = "bootstrap-self-test")) { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: remove the condition + | + = note: no expected values for `feature` + = help: consider adding `bootstrap-self-test` as a feature in `Cargo.toml` + = note: see for more information about checking conditional configuration + = note: `#[warn(unexpected_cfgs)]` on by default + +warning: unused import: `std::env` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:6:5 + | +6 | use std::env; + | ^^^^^^^^ + +warning: unused import: `std::fs` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:7:5 + | +7 | use std::fs; + | ^^^^^^^ + +warning: unused import: `Path` + --> standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs:1:26 + | +1 | use std::path::{PathBuf, Path}; + | ^^^^ + +warning: unused import: `crate::local_flags::LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs:4:5 + | +4 | use crate::local_flags::LocalFlags; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `Path` + --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:2:26 + | +2 | use std::path::{PathBuf, Path}; + | ^^^^ + +warning: unused import: `std::collections::HashMap` + --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:3:5 + | +3 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `serde_derive::Deserialize` + --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:4:5 + | +4 | use serde_derive::Deserialize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused import: `crate::dry_run::DryRun` + --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:6:5 + | +6 | use crate::dry_run::DryRun; + | ^^^^^^^^^^^^^^^^^^^^^^ + +error[E0609]: no field `subcommand` on type `&mut LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:5:24 + | +5 | config.cmd = flags.subcommand.take(); + | ^^^^^^^^^^ unknown field + | + = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + +error[E0609]: no field `incremental` on type `&mut LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:6:32 + | +6 | config.incremental = flags.incremental; + | ^^^^^^^^^^^ unknown field + | + = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + +error[E0609]: no field `dry_run` on type `&mut LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:7:28 + | +7 | config.dry_run = flags.dry_run; + | ^^^^^^^ unknown field + | + = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + +error[E0609]: no field `verbose` on type `&mut LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:8:33 + | +8 | config.verbose = Some(flags.verbose); + | ^^^^^^^ unknown field + | + = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + +error[E0609]: no field `stage` on type `&mut LocalFlags` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:9:26 + | +9 | config.stage = flags.stage.unwrap_or_default(); + | ^^^^^ unknown field + | + = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + +error[E0308]: mismatched types + --> standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs:11:34 + | +11 | config.channel = channel; + | -------------- ^^^^^^^ expected `Option`, found `String` + | | + | expected due to the type of this binding + | + = note: expected enum `std::option::Option` + found struct `std::string::String` +help: try wrapping the expression in `Some` + | +11 | config.channel = Some(channel); + | +++++ + + +error[E0308]: mismatched types + --> standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs:17:109 + | +17 | config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(config.channel == "dev"); + | -------------- ^^^^^ expected `Option`, found `&str` + | | + | expected because this is `std::option::Option` + | + = note: expected enum `std::option::Option` + found reference `&'static str` + +warning: unused variable: `config` + --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:48:22 + | +48 | fn apply_test_config(config: &mut ParsedConfig, toml: &mut LocalTomlConfig) { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_config` + | + = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default + +Some errors have detailed explanations: E0308, E0412, E0422, E0433, E0609. +For more information about an error, try `rustc --explain E0308`. +warning: `bootstrap-config-utils` (lib) generated 22 warnings +error: could not compile `bootstrap-config-utils` (lib) due to 29 previous errors; 22 warnings emitted diff --git a/standalonex/src/bootstrap/src/core/config_utils/report2.txt b/standalonex/src/bootstrap/src/core/config_utils/report2.txt new file mode 100644 index 00000000..81949fd2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/report2.txt @@ -0,0 +1,253 @@ + 1 | | + 1 | +++++ + + 1 | | + 1 | ^^^^ ^^^^^^^ + 1 | ^^^^^^^ ^^^^ + 1 | ^^^^ + 1 | ^^^^^^^^^^^^^^^ + 1 | ^^^^^^^^^^^^^^^^^^^^^^ + 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 1 10 + pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &LocalFlags) { + 1 10 + pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut LocalTomlConfig, flags: &crate::LocalFlags) { + 1 10 + pub fn parse_inner_build(config: &mut ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + 1 11 | config.channel = channel; + 1 11 | config.channel = Some(channel); + 1 11 - let crate::LocalBuild { + 1 11 | let crate::LocalBuild { + 1 11 + let LocalBuild { + 1 13 - pub(crate) fn get_toml(file: &Path) -> Result { + 1 13 | pub(crate) fn get_toml(file: &Path) -> Result { + 1 13 + pub(crate) fn get_toml(file: &Path) -> Result { + 1 13 - pub fn dry_run(config: &crate::ParsedConfig) -> bool { + 1 13 | pub fn dry_run(config: &crate::ParsedConfig) -> bool { + 1 13 + pub fn dry_run(config: &ParsedConfig) -> bool { + 1 14 - crate::ParsedConfig { + 1 14 | crate::ParsedConfig { + 1 14 + ParsedConfig { + 1 15 | crate::DryRun::Disabled => false, + 1 17 | config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(config.channel == "dev"); + 1 19 - .and_then(|table: toml::Value| crate::LocalTomlConfig::deserialize(table)) + 1 19 | .and_then(|table: toml::Value| crate::LocalTomlConfig::deserialize(table)) + 1 19 + .and_then(|table: toml::Value| LocalTomlConfig::deserialize(table)) + 1 1 + pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &LocalTomlConfig) { + 1 1 + pub fn parse_inner_stage0(config: &mut ParsedConfig, toml: &crate::LocalTomlConfig) { + 1 1 + use crate::parsed_config::ParsedConfig; + 1 1 | use std::path::{PathBuf, Path}; + 1 25 | config.config = Some(if cfg!(not(feature = "bootstrap-self-test")) { + 1 2 + use crate::dry_run::DryRun; + 1 2 + use crate::local_ci_config::LocalCiConfig; + 1 2 + use crate::parsed_config::ParsedConfig; + 1 2 + use crate::target_selection::TargetSelection; + 1 2 | use std::path::PathBuf; + 1 3 | use serde::Deserialize; + 1 3 | use std::path::{Path, PathBuf}; + 1 42 - build: crate::TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + 1 42 | build: crate::TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + 1 42 + build: TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + 1 42 - pub dry_run: crate::DryRun, + 1 42 | pub dry_run: crate::DryRun, + 1 42 + pub dry_run: DryRun, + 1 48 | fn apply_test_config(config: &mut ParsedConfig, toml: &mut LocalTomlConfig) { + 1 4 | use crate::get_toml; + 1 4 | use crate::local_flags::LocalFlags; + 1 4 | use serde_derive::Deserialize; + 1 4 | use std::env; + 1 51 - ci: Some(crate::LocalCiConfig { + 1 51 | ci: Some(crate::LocalCiConfig { + 1 51 + ci: Some(LocalCiConfig { + 1 5 | config.cmd = flags.subcommand.take(); + 1 5 + use crate::local_build::LocalBuild; + 1 5 + use crate::local_flags::LocalFlags; + 1 5 + use crate::local_toml_config::LocalTomlConfig; + 1 5 | use crate::local_toml_config::LocalTomlConfig; + 1 5 + use crate::parsed_config::ParsedConfig; + 1 5 | use serde_derive::Deserialize; + 1 5 | use toml; + 1 6 | config.incremental = flags.incremental; + 1 6 + pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + 1 6 + pub fn get_builder_toml(config: &ParsedConfig, build_name: &str) -> Result { + 1 6 | use crate::dry_run::DryRun; + 1 6 | use std::env; + 1 7 | config.dry_run = flags.dry_run; + 1 7 - pub fn default_opts() -> crate::ParsedConfig { + 1 7 | pub fn default_opts() -> crate::ParsedConfig { + 1 7 + pub fn default_opts() -> ParsedConfig { + 1 7 | use std::fs; + 1 8 | config.verbose = Some(flags.verbose); + 1 8 - return Ok(crate::LocalTomlConfig::default()); + 1 8 | return Ok(crate::LocalTomlConfig::default()); + 1 8 + return Ok(LocalTomlConfig::default()); + 1 8 | use crate::parse_inner_stage0; + 1 8 | use std::fs; + 1 9 | config.stage = flags.stage.unwrap_or_default(); + 1 Compiling bootstrap-config-utils v0.1.0 (/data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/vendor/rust/rust-bootstrap-nix/standalonex/src/bootstrap/src/core/config_utils) + 1 | ^^^^^^ could not find `DryRun` in the crate root + 1 | ^^^^^^^^^^^^^^^ could not find `LocalTomlConfig` in the crate root + 1 | ^^^^^^^^^^^^^^^ could not find `LocalTomlConfig` in the crate root + 1 | ^^^^^^^^^^^^^^^ could not find `TargetSelection` in the crate root + 1 error: could not compile `bootstrap-config-utils` (lib) due to 29 previous errors; 22 warnings emitted + 1 error[E0412]: cannot find type `DryRun` in the crate root + 1 error[E0412]: cannot find type `LocalFlags` in the crate root + 1 error[E0412]: cannot find type `LocalTomlConfig` in this scope + 1 error[E0412]: cannot find type `ParsedConfig` in this scope + 1 error[E0422]: cannot find struct, variant or union type `LocalBuild` in the crate root + 1 error[E0422]: cannot find struct, variant or union type `LocalCiConfig` in the crate root + 1 error[E0422]: cannot find struct, variant or union type `ParsedConfig` in the crate root + 1 error[E0433]: failed to resolve: could not find `TargetSelection` in the crate root + 1 error[E0609]: no field `dry_run` on type `&mut LocalFlags` + 1 error[E0609]: no field `incremental` on type `&mut LocalFlags` + 1 error[E0609]: no field `stage` on type `&mut LocalFlags` + 1 error[E0609]: no field `subcommand` on type `&mut LocalFlags` + 1 error[E0609]: no field `verbose` on type `&mut LocalFlags` + 1 | expected because this is `std::option::Option` + 1 | expected due to the type of this binding + 1 | -------------- ^^^^^ expected `Option`, found `&str` + 1 | -------------- ^^^^^^^ expected `Option`, found `String` + 1 For more information about an error, try `rustc --explain E0308`. + 1 found reference `&'static str` + 1 found struct `std::string::String` + 1 = help: consider adding `bootstrap-self-test` as a feature in `Cargo.toml` + 1 help: consider importing this enum + 1 | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_config` + 1 help: if you import `DryRun`, refer to it directly + 1 help: if you import `LocalBuild`, refer to it directly + 1 help: if you import `LocalCiConfig`, refer to it directly + 1 help: if you import `LocalFlags`, refer to it directly + 1 help: if you import `TargetSelection`, refer to it directly + 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: remove the condition + 1 help: try wrapping the expression in `Some` + 1 = note: expected enum `std::option::Option` + 1 = note: expected enum `std::option::Option` + 1 note: for more details see https://doc.rust-lang.org/cargo/reference/resolver.html#resolver-versions + 1 = note: no expected values for `feature` + 1 = note: see for more information about checking conditional configuration + 1 note: to keep the current resolver, specify `workspace.resolver = "1"` in the workspace root's manifest + 1 note: to use the edition 2021 resolver, specify `workspace.resolver = "2"` in the workspace root's manifest + 1 = note: `#[warn(unexpected_cfgs)]` on by default + 1 = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default + 1 = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default + 1 | ^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^ not found in the crate root + 1 | ^^^^^^^^^^^^^^^ not found in this scope + 1 | ^^^^^^^^^^^^ not found in this scope + 1 Some errors have detailed explanations: E0308, E0412, E0422, E0433, E0609. + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs:1:26 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs:4:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs:2:44 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs:2:65 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:14:12 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:42:23 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:51:25 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs:7:33 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:13:32 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:15:16 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:16:16 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs:16:43 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:2:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:6:41 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:6:90 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs:8:26 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:13:54 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:19:47 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:3:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs:5:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:2:17 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:3:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/lib.rs:5:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:2:26 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:3:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:42:25 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:4:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs:6:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:110 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:46 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:10:78 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:11:16 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:6:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs:7:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:5:24 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:6:32 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:7:28 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:8:33 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs:9:26 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs:1:47 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs:1:75 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:25:42 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:4:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs:8:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:3:17 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:4:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:48:22 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:5:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/parse.rs:8:5 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs:11:34 + 1 --> standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs:17:109 + 1 | ^^^^^^^ unknown field + 1 | ^^^^^^^^^^^ unknown field + 1 | ^^^^^^^ unknown field + 1 | ^^^^^ unknown field + 1 | ^^^^^^^^^^ unknown field + 1 warning: `bootstrap-config-utils` (lib) generated 22 warnings + 1 warning: unexpected `cfg` condition value: `bootstrap-self-test` + 1 warning: unused import: `crate::dry_run::DryRun` + 1 warning: unused import: `crate::get_toml` + 1 warning: unused import: `crate::local_flags::LocalFlags` + 1 warning: unused import: `crate::local_toml_config::LocalTomlConfig` + 1 warning: unused import: `crate::parse_inner_stage0` + 1 warning: unused import: `serde::Deserialize` + 1 warning: unused import: `std::path::PathBuf` + 1 warning: unused import: `toml` + 1 warning: unused variable: `config` + 1 warning: virtual workspace defaulting to `resolver = "1"` despite one or more workspace members being on edition 2021 which implies `resolver = "2"` + 2 | ^^^^ + 2 | ^^^^^^^ + 2 | ^^^^^^^^ + 2 | ^^^^^^^^^^^^^^^^^^ + 2 16 | crate::DryRun::SelfCheck | crate::DryRun::UserSelected => true, + 2 1 - pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { + 2 1 | pub fn parse_inner_stage0(config: &mut crate::ParsedConfig, toml: &crate::LocalTomlConfig) { + 2 1 + use crate::local_toml_config::LocalTomlConfig; + 2 1 + use crate::local_toml_config::LocalTomlConfig; + 2 1 + use crate::parsed_config::ParsedConfig; + 2 2 | fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig); + 2 2 + use crate::local_toml_config::LocalTomlConfig; + 2 2 + use crate::parsed_config::ParsedConfig; + 2 2 | use std::path::{PathBuf, Path}; + 2 3 | use std::collections::HashMap; + 2 6 - pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + 2 6 | pub fn get_builder_toml(config: &crate::ParsedConfig, build_name: &str) -> Result { + 2 | ^^^^^^ could not find `DryRun` in the crate root + 2 error[E0308]: mismatched types + 2 error[E0433]: failed to resolve: could not find `LocalTomlConfig` in the crate root + 2 warning: unused import: `Path` + 2 warning: unused import: `serde_derive::Deserialize` + 2 warning: unused imports: `PathBuf` and `Path` + 2 warning: unused import: `std::collections::HashMap` + 2 warning: unused import: `std::env` + 2 warning: unused import: `std::fs` + 3 10 - pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + 3 10 | pub fn parse_inner_build(config: &mut crate::ParsedConfig, toml: &mut crate::LocalTomlConfig, flags: &crate::LocalFlags) { + 3 error[E0433]: failed to resolve: could not find `DryRun` in the crate root + 4 error[E0412]: cannot find type `LocalTomlConfig` in the crate root + 5 | ^^^^^^^^^^^^^^^^^^^^^^^^^ + 5 error[E0412]: cannot find type `ParsedConfig` in the crate root + 5 = note: available fields are: `set`, `jobs`, `build_dir`, `skip_stage0_validation`, `host` ... and 3 others + 6 help: if you import `LocalTomlConfig`, refer to it directly + 6 help: if you import `ParsedConfig`, refer to it directly + 18 help: consider importing this struct + 51 + 75 | + 78 | diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs new file mode 100644 index 00000000..b8c6243b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/build_config.rs @@ -0,0 +1,60 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; + +pub struct BuildConfigApplicator; + +impl ConfigApplicator for BuildConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + let build_config = toml.build.clone().unwrap_or_default(); + + config.jobs = config.jobs.or(build_config.jobs).or(Some(0)); + + if let Some(file_build) = build_config.build { + config.build_triple = Some(file_build); + }; + + // config.out_dir = flags.build_dir.or_else(|| build_config.build_dir.map(PathBuf::from)); // flags is not available here + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + // if !config.out.is_absolute() { + // // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + // config.out = absolute(&config.out).expect("can't make empty path absolute"); + // } + + config.hosts = if let Some(file_host) = build_config.host { + file_host + } else { + vec![config.build_triple.clone().unwrap_or_default()] + }; + config.targets = if let Some(file_target) = build_config.target { + file_target + } else { + config.hosts.clone() + }; + + config.nodejs = build_config.nodejs; + config.npm = build_config.npm; + config.gdb = build_config.gdb; + config.lldb = build_config.lldb; + config.python = build_config.python; + config.reuse = build_config.reuse; + config.submodules = build_config.submodules; + config.android_ndk = build_config.android_ndk; + config.bootstrap_cache_path = build_config.bootstrap_cache_path; + config.low_priority = build_config.low_priority; + config.compiler_docs = build_config.compiler_docs; + config.library_docs_private_items = build_config.library_docs_private_items; + config.docs_minification = build_config.docs_minification; + config.docs = build_config.docs; + config.locked_deps = build_config.locked_deps; + config.vendor = build_config.vendor; + config.full_bootstrap = build_config.full_bootstrap; + config.extended = build_config.extended; + config.tools = build_config.tools; + config.verbose = build_config.verbose; + config.sanitizers = build_config.sanitizers; + config.profiler = build_config.profiler; + config.cargo_native_static = build_config.cargo_native_static; + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/ci_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/ci_config.rs new file mode 100644 index 00000000..31e856b1 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/ci_config.rs @@ -0,0 +1,18 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; + +pub struct CiConfigApplicator; + +impl ConfigApplicator for CiConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + let ci_config = toml.ci.clone().unwrap_or_default(); + config.channel_file = ci_config.channel_file; + config.version_file = ci_config.version_file; + config.tools_dir = ci_config.tools_dir; + config.llvm_project_dir = ci_config.llvm_project_dir; + config.gcc_dir = ci_config.gcc_dir; + +// config.change_id = toml.change_id.inner; + } +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs b/standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs new file mode 100644 index 00000000..8612eaa7 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/config_applicator.rs @@ -0,0 +1,5 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +pub trait ConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig); +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs b/standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs new file mode 100644 index 00000000..6b0c1836 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/default_opts.rs @@ -0,0 +1,65 @@ +//use bootstrap::prelude::*; +use std::path::PathBuf; +use crate::target_selection::TargetSelection; +use crate::local_ci_config::LocalCiConfig; +use crate::parsed_config::ParsedConfig; + +use std::env; + +use std::io::IsTerminal; + +pub fn default_opts() -> ParsedConfig { + let src_path = { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + manifest_dir.parent().unwrap().parent().unwrap().to_owned() + }; + + ParsedConfig { + bypass_bootstrap_lock: false, + llvm_optimize: Some(true), + ninja_in_file: Some(true), + llvm_static_stdcpp: Some(false), + llvm_libzstd: Some(false), + backtrace: Some(true), +// rust_optimize: RustOptimize::Bool(true), + rust_optimize_tests: Some(true), + rust_randomize_layout: false, + submodules: None, + docs: Some(true), + docs_minification: Some(true), + rust_rpath: Some(true), + rust_strip: Some(false), + channel: Some("dev".to_string()), + codegen_tests: Some(true), + rust_dist_src: Some(true), + rust_codegen_backends: vec!["llvm".to_owned()], + deny_warnings: Some(true), + bindir: Some("bin".into()), + dist_include_mingw_linker: Some(true), + dist_compression_profile: Some("fast".into()), + + stdout_is_tty: Some(std::io::stdout().is_terminal()), + stderr_is_tty: Some(std::io::stderr().is_terminal()), + + // set by build.rs + build: TargetSelection::from_user(&env::var("BUILD_TRIPLE").unwrap()), + + src: src_path.clone(), + out: PathBuf::from("build"), + + // This is needed by codegen_ssa on macOS to ship `llvm-objcopy` aliased to + // `rust-objcopy` to workaround bad `strip`s on macOS. + llvm_tools_enabled: true, + + ci: Some(LocalCiConfig { + channel_file: Some(src_path.join("src/ci/channel")), + version_file: Some(src_path.join("src/version")), + tools_dir: Some(src_path.join("src/tools")), + llvm_project_dir: Some(src_path.join("src/llvm-project")), + gcc_dir: Some(src_path.join("src/gcc")), + }), + + ..Default::default() + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs b/standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs new file mode 100644 index 00000000..28f637ee --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/dry_run.rs @@ -0,0 +1,17 @@ +use crate::parsed_config::ParsedConfig; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[derive(Default)] +pub enum DryRun { + #[default] + Disabled, + SelfCheck, + UserSelected, +} + +pub fn dry_run(config: &ParsedConfig) -> bool { + match config.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs b/standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs new file mode 100644 index 00000000..8172da35 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/get_builder_toml.rs @@ -0,0 +1,21 @@ + +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::target_selection::TargetSelection; +use crate::get_toml; +use crate::dry_run::DryRun; + +const BUILDER_CONFIG_FILENAME: &str = "config.toml"; + +pub fn get_builder_toml(config: &ParsedConfig, build_name: &str) -> Result { + if config.dry_run != DryRun::Disabled { + return Ok(LocalTomlConfig::default()); + } + + let TargetSelection(ref build_triple) = config.build; + let builder_config_path = + config.out.join(build_triple).join(build_name).join(BUILDER_CONFIG_FILENAME); + // Assuming get_toml will also be moved and called as a standalone function + // For now, I'll keep it as Config::get_toml and fix it later when get_toml is moved. + get_toml::get_toml(&builder_config_path) +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs b/standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs new file mode 100644 index 00000000..3957e4dc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/get_toml.rs @@ -0,0 +1,20 @@ +use std::path::Path; +use std::fs; +use serde::Deserialize; + +use crate::local_toml_config::LocalTomlConfig; + +#[cfg(test)] +pub(crate) fn get_toml(_: &Path) -> Result { + Ok(LocalTomlConfig::default()) +} + +#[cfg(not(test))] +pub(crate) fn get_toml(file: &Path) -> Result { + let contents = + fs::read_to_string(file).unwrap_or_else(|_| panic!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + toml::from_str(&contents) + .and_then(|table: toml::Value| LocalTomlConfig::deserialize(table)) +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/install_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/install_config.rs new file mode 100644 index 00000000..a30315f9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/install_config.rs @@ -0,0 +1,40 @@ +use std::path::PathBuf; +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; +use serde_derive::Deserialize; + + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct Install { + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: Option, + pub libdir: Option, + pub mandir: Option, +} + +pub struct InstallConfigApplicator; + +impl ConfigApplicator for InstallConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + if let Some(install) = &toml.install { + let Install { prefix, sysconfdir, docdir, bindir, libdir, mandir, datadir } = install; + config.prefix = prefix.clone(); + config.sysconfdir = sysconfdir.clone(); + config.datadir = datadir.clone(); + config.docdir = docdir.clone(); + // Handle bindir specifically, as it's not an Option in Config + if let Some(b) = bindir { + config.bindir = Some(b.clone()); + } else if let Some(p) = &config.prefix { + config.bindir = Some(p.join("bin")); + } + config.libdir = libdir.clone(); + config.mandir = mandir.clone(); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/lib.rs b/standalonex/src/bootstrap/src/core/config_utils/src/lib.rs new file mode 100644 index 00000000..63c7659b --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/lib.rs @@ -0,0 +1,82 @@ +// This will be the lib.rs for the new bootstrap-config-utils crate + +pub mod default_opts; +pub mod get_builder_toml; +pub mod get_toml; +pub mod parse; +pub mod parse_inner_flags; +pub mod parse_inner_src; +pub mod parse_inner_out; +pub mod parse_inner_toml; +pub mod parse_inner_build; +pub mod dry_run; +pub mod try_run; +pub mod ci_config; +pub mod build_config; +pub mod install_config; +pub mod config_applicator; +pub mod llvm_assertions_config; +pub mod rust_channel_git_hash_config; +pub mod nix_config; +pub mod local_build; +pub mod local_ci_config; +pub mod local_dist; +pub mod local_flags; +pub mod local_llvm; +pub mod local_rust; +pub mod local_target_config; +pub mod local_toml_config; +pub mod local_nix_config; +pub mod parsed_config; +pub mod target_selection; + +#[cfg(test)] +mod config_parsing_tests { + use super::*; + use std::path::PathBuf; + use crate::parse::parse; + use crate::local_flags::LocalFlags; + use crate::dry_run::DryRun; + + #[test] + fn test_parse_example_configs() { + let config_files = [ + "../../../../../../config.toml", // Main config.toml + "../../../../../../standalonex/config.toml", + "../../../../../../standalonex/src/bootstrap/stage0/config.toml", + "../../../../../../standalonex/src/bootstrap/defaults/config.compiler.toml", + "../../../../../../standalonex/src/bootstrap/defaults/config.dist.toml", + "../../../../../../standalonex/src/bootstrap/defaults/config.library.toml", + "../../../../../../standalonex/src/bootstrap/defaults/config.tools.toml", + ]; + + for &file_path_str in &config_files { + let file_path = PathBuf::from(file_path_str); + println!("Testing config file: {}", file_path.display()); + + let flags = LocalFlags::default(); // Create a default LocalFlags + + // Attempt to parse the config file + // This will panic if parsing fails, which is what we want to catch in a test + let parsed_config = parse(flags); + + // Add some basic assertions to check if parsing was successful and values are as expected + // These assertions will need to be tailored to the actual content of each config file + // For now, we'll just check if some common fields are not their default values if expected. + + // Example assertion for standalonex/config.toml + if file_path_str.contains("standalonex/config.toml") { + // Assuming change-id is parsed into parsed_config.change_id + // This requires `change_id` to be part of ParsedConfig and LocalTomlConfig + // and handled by an applicator. + // For now, just check if it doesn't panic. + } + + // Assert that dry_run is Disabled by default flags + assert_eq!(parsed_config.dry_run, DryRun::Disabled); + + // Assert that out path is not empty + assert!(!parsed_config.out.to_str().unwrap().is_empty()); + } + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/llvm_assertions_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/llvm_assertions_config.rs new file mode 100644 index 00000000..fbdcea98 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/llvm_assertions_config.rs @@ -0,0 +1,11 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; + +pub struct LlvmAssertionsConfigApplicator; + +impl ConfigApplicator for LlvmAssertionsConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + config.llvm_assertions = Some(toml.llvm.as_ref().and_then(|llvm| llvm.assertions).unwrap_or(false)); + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_build.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_build.rs new file mode 100644 index 00000000..0df23f0a --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_build.rs @@ -0,0 +1,55 @@ + +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalBuild { + pub build: Option, + pub host: Option>, + pub target: Option>, + pub build_dir: Option, + pub cargo: Option, + pub rustc: Option, + pub rustfmt: Option, + pub cargo_clippy: Option, + pub docs: Option, + pub compiler_docs: Option, + pub library_docs_private_items: Option, + pub docs_minification: Option, + pub submodules: Option, + pub gdb: Option, + pub lldb: Option, + pub nodejs: Option, + pub npm: Option, + pub python: Option, + pub reuse: Option, + pub locked_deps: Option, + pub vendor: Option, + pub full_bootstrap: Option, + pub bootstrap_cache_path: Option, + pub extended: Option, + pub tools: Option>, + pub verbose: Option, + pub sanitizers: Option, + pub profiler: Option, + pub cargo_native_static: Option, + pub low_priority: Option, + pub configure_args: Option>, + pub local_rebuild: Option, + pub print_step_timings: Option, + pub print_step_rusage: Option, + pub check_stage: Option, + pub doc_stage: Option, + pub build_stage: Option, + pub test_stage: Option, + pub install_stage: Option, + pub dist_stage: Option, + pub bench_stage: Option, + pub patch_binaries_for_nix: Option, + pub metrics: Option, + pub android_ndk: Option, + pub optimized_compiler_builtins: Option, + pub jobs: Option, + pub compiletest_diff_tool: Option, + pub src: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_ci_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_ci_config.rs new file mode 100644 index 00000000..5bb93fa6 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_ci_config.rs @@ -0,0 +1,11 @@ +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalCiConfig { + pub channel_file: Option, + pub version_file: Option, + pub tools_dir: Option, + pub llvm_project_dir: Option, + pub gcc_dir: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_dist.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_dist.rs new file mode 100644 index 00000000..0e09680f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_dist.rs @@ -0,0 +1,14 @@ +use serde_derive::Deserialize; +use std::path::PathBuf; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalDist { + pub sign_folder: Option, + pub upload_addr: Option, + pub compression_formats: Option>, + pub compression_profile: Option, + pub src_tarball: Option, + pub include_mingw_linker: Option, + pub vendor: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_flags.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_flags.rs new file mode 100644 index 00000000..6a484d9c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_flags.rs @@ -0,0 +1,14 @@ +use std::path::PathBuf; +use crate::target_selection::TargetSelection; + +#[derive(Default)] +pub struct LocalFlags { + pub set: Vec, + pub jobs: Option, + pub build_dir: Option, + pub skip_stage0_validation: bool, + pub host: Vec, + pub target: Vec, + pub src: Option, + pub config: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_llvm.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_llvm.rs new file mode 100644 index 00000000..cfb85492 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_llvm.rs @@ -0,0 +1,37 @@ + +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalLlvm { + pub optimize: Option, + pub thin_lto: Option, + pub release_debuginfo: Option, + pub assertions: Option, + pub tests: Option, + pub enzyme: Option, + pub plugins: Option, + pub ccache: Option, + pub static_libstdcpp: Option, + pub libzstd: Option, + pub ninja: Option, + pub targets: Option>, + pub experimental_targets: Option>, + pub link_jobs: Option, + pub link_shared: Option, + pub version_suffix: Option, + pub clang_cl: Option, + pub cflags: Option, + pub cxxflags: Option, + pub ldflags: Option, + pub use_libcxx: Option, + pub use_linker: Option, + pub allow_old_toolchain: Option, + pub offload: Option, + pub polly: Option, + pub clang: Option, + pub enable_warnings: Option, + pub download_ci_llvm: Option, + pub build_config: Option, + pub enable_projects: Option>, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_nix_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_nix_config.rs new file mode 100644 index 00000000..fd163100 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_nix_config.rs @@ -0,0 +1,12 @@ +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize, Clone)] +pub struct LocalNixConfig { + pub nixpkgs_path: Option, + pub rust_overlay_path: Option, + pub rust_bootstrap_nix_path: Option, + pub configuration_nix_path: Option, + pub rust_src_flake_path: Option, + pub rust_bootstrap_nix_flake_ref: Option, + pub rust_src_flake_ref: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_rust.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_rust.rs new file mode 100644 index 00000000..d3713cde --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_rust.rs @@ -0,0 +1,61 @@ + +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalRust { + pub optimize: Option, + pub debug: Option, + pub codegen_units: Option, + pub codegen_units_std: Option, + pub rustc_debug_assertions: Option, + pub std_debug_assertions: Option, + pub overflow_checks: Option, + pub overflow_checks_std: Option, + pub debug_logging: Option, + pub debuginfo_level: Option, + pub debuginfo_level_rustc: Option, + pub debuginfo_level_std: Option, + pub debuginfo_level_tools: Option, + pub debuginfo_level_tests: Option, + pub backtrace: Option, + pub incremental: Option, + pub parallel_compiler: Option, + pub randomize_layout: Option, + pub default_linker: Option, + pub channel: Option, + pub description: Option, + pub musl_root: Option, + pub rpath: Option, + pub verbose_tests: Option, + pub optimize_tests: Option, + pub codegen_tests: Option, + pub omit_git_hash: Option, + pub dist_src: Option, + pub save_toolstates: Option, + pub codegen_backends: Option>, + pub lld: Option, + pub llvm_tools: Option, + pub llvm_bitcode_linker: Option, + pub deny_warnings: Option, + pub backtrace_on_ice: Option, + pub verify_llvm_ir: Option, + pub thin_lto_import_instr_limit: Option, + pub remap_debuginfo: Option, + pub jemalloc: Option, + pub test_compare_mode: Option, + pub llvm_libunwind: Option, + pub control_flow_guard: Option, + pub ehcont_guard: Option, + pub new_symbol_mangling: Option, + pub profile_generate: Option, + pub profile_use: Option, + pub download_rustc: Option, + pub lto: Option, + pub validate_mir_opts: Option, + pub frame_pointers: Option, + pub stack_protector: Option, + pub strip: Option, + pub lld_mode: Option, + pub std_features: Option>, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_target_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_target_config.rs new file mode 100644 index 00000000..beda4828 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_target_config.rs @@ -0,0 +1,28 @@ + +use serde_derive::Deserialize; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalTargetConfig { + pub llvm_config: Option, + pub llvm_has_rust_patches: Option, + pub llvm_filecheck: Option, + pub llvm_libunwind: Option, + pub no_std: Option, + pub cc: Option, + pub cxx: Option, + pub ar: Option, + pub ranlib: Option, + pub linker: Option, + pub crt_static: Option, + pub musl_root: Option, + pub musl_libdir: Option, + pub wasi_root: Option, + pub qemu_rootfs: Option, + pub runner: Option>, + pub sanitizers: Option, + pub profiler: Option, + pub rpath: Option, + pub codegen_backends: Option>, + pub split_debuginfo: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/local_toml_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/local_toml_config.rs new file mode 100644 index 00000000..92bc4c7f --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/local_toml_config.rs @@ -0,0 +1,23 @@ +use serde_derive::Deserialize; +use crate::local_ci_config::LocalCiConfig; +use crate::local_build::LocalBuild; +use crate::local_llvm::LocalLlvm; +use crate::local_rust::LocalRust; +use crate::local_target_config::LocalTargetConfig; +use crate::local_dist::LocalDist; +use crate::install_config; +use crate::local_nix_config::LocalNixConfig; + +#[derive(Debug, Default, Deserialize)] +#[derive(Clone)] +pub struct LocalTomlConfig { + pub ci: Option, + pub build: Option, + pub llvm: Option, + pub rust: Option, + pub target: Option>, + pub install: Option, + pub dist: Option, + pub nix: Option, + // ... other fields will go here +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/nix_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/nix_config.rs new file mode 100644 index 00000000..decd3cfb --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/nix_config.rs @@ -0,0 +1,19 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; + +pub struct NixConfigApplicator; + +impl ConfigApplicator for NixConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + let nix_config = toml.nix.clone().unwrap_or_default(); + + config.nixpkgs_path = nix_config.nixpkgs_path; + config.rust_overlay_path = nix_config.rust_overlay_path; + config.rust_bootstrap_nix_path = nix_config.rust_bootstrap_nix_path; + config.configuration_nix_path = nix_config.configuration_nix_path; + config.rust_src_flake_path = nix_config.rust_src_flake_path; + config.rust_bootstrap_nix_flake_ref = nix_config.rust_bootstrap_nix_flake_ref; + config.rust_src_flake_ref = nix_config.rust_src_flake_ref; + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse.rs new file mode 100644 index 00000000..4da0e7a9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse.rs @@ -0,0 +1,56 @@ + +use crate::parsed_config::ParsedConfig; + +use crate::parse_inner_flags; +use crate::parse_inner_toml; +use crate::parse_inner_src; +use crate::parse_inner_out; +use crate::config_applicator::ConfigApplicator; +use crate::ci_config; +use crate::build_config; +use crate::install_config; +use crate::llvm_assertions_config; +use crate::rust_channel_git_hash_config; +use crate::nix_config; + +use crate::local_flags::LocalFlags; +use crate::local_toml_config::LocalTomlConfig; +use crate::get_toml; + +pub fn parse(mut flags: LocalFlags) -> ParsedConfig { + let mut config = ParsedConfig::default(); + + // Set flags. + parse_inner_flags::parse_inner_flags(&mut config, &mut flags); + + // Infer the rest of the configuration. + let build_src_from_toml = None; // This needs to be handled differently if it's coming from toml.build.src + parse_inner_src::parse_inner_src(&mut config, &flags, &build_src_from_toml); + + parse_inner_out::parse_inner_out(&mut config); + + let toml = parse_inner_toml::parse_inner_toml(&mut config, &flags, get_toml::get_toml); + + // Apply various configuration applicators + let mut applicators: Vec> = Vec::new(); + applicators.push(Box::new(ci_config::CiConfigApplicator)); + applicators.push(Box::new(build_config::BuildConfigApplicator)); + applicators.push(Box::new(install_config::InstallConfigApplicator)); + applicators.push(Box::new(llvm_assertions_config::LlvmAssertionsConfigApplicator)); + applicators.push(Box::new(rust_channel_git_hash_config::RustChannelGitHashConfigApplicator)); + applicators.push(Box::new(nix_config::NixConfigApplicator)); + + for applicator in applicators.iter() { + applicator.apply_to_config(&mut config, &toml); + } + + config +} + +fn apply_test_config(_config: &mut ParsedConfig, toml: &mut LocalTomlConfig) { + if cfg!(test) { + let build = toml.build.get_or_insert_with(Default::default); + build.rustc = build.rustc.take().or(std::env::var_os("RUSTC").map(|p| p.into())); + build.cargo = build.cargo.take().or(std::env::var_os("CARGO").map(|p| p.into())); + } +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs new file mode 100644 index 00000000..b721f3a3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_build.rs @@ -0,0 +1,78 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::local_flags::LocalFlags; +use crate::local_build::LocalBuild; +use crate::dry_run::DryRun; + +pub fn parse_inner_build(config: &mut ParsedConfig, toml: &mut LocalTomlConfig, _flags: &LocalFlags) { + let LocalBuild { + build: _, + host: _, + target: _, + build_dir: _, + cargo: _, + rustc: _, + rustfmt: _, + cargo_clippy, + docs: _, + compiler_docs: _, + library_docs_private_items: _, + docs_minification: _, + submodules, + gdb, + lldb, + nodejs, + npm, + python, + reuse, + locked_deps: _, + vendor: _, + full_bootstrap: _, + bootstrap_cache_path, + extended: _, + tools, + verbose: _, + sanitizers: _, + profiler: _, + cargo_native_static: _, + low_priority: _, + configure_args: _, + local_rebuild: _, + print_step_timings: _, + print_step_rusage: _, + check_stage: _, + doc_stage: _, + build_stage: _, + test_stage: _, + install_stage: _, + dist_stage: _, + bench_stage: _, + patch_binaries_for_nix, + // This field is only used by bootstrap.py + metrics: _, + android_ndk, + optimized_compiler_builtins: _, + jobs: _, + compiletest_diff_tool: _, + src: _build_src_from_toml, + } = toml.build.clone().unwrap_or_default(); + + config.initial_cargo_clippy = cargo_clippy; + + if config.dry_run != DryRun::Disabled { + let dir = config.out.join("tmp-dry-run"); + config.out = dir; + } + + config.nodejs = nodejs; + config.npm = npm; + config.gdb = gdb; + config.lldb = lldb; + config.python = python; + config.reuse = reuse; + config.submodules = submodules; + config.android_ndk = android_ndk; + config.bootstrap_cache_path = bootstrap_cache_path; + config.tools = tools; + config.patch_binaries_for_nix = patch_binaries_for_nix; +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs new file mode 100644 index 00000000..fb76ab99 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_flags.rs @@ -0,0 +1,6 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_flags::LocalFlags; + +pub fn parse_inner_flags(_config: &mut ParsedConfig, _flags: &mut LocalFlags) { + // These fields are no longer part of LocalFlags and are handled elsewhere. +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_out.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_out.rs new file mode 100644 index 00000000..3fead2b3 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_out.rs @@ -0,0 +1,12 @@ +use crate::parsed_config::ParsedConfig; +use std::path::{Path, PathBuf}; +use std::env; + +pub fn parse_inner_out(config: &mut ParsedConfig) { + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = env::var_os("CARGO_TARGET_DIR") + .map(|s| Path::new(&s).parent().unwrap().to_path_buf()) + .unwrap_or_else(|| PathBuf::from("target/test_output")); // Provide a default for tests + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_src.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_src.rs new file mode 100644 index 00000000..8daf561c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_src.rs @@ -0,0 +1,16 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_flags::LocalFlags; +use std::path::PathBuf; +use std::env; + +pub fn parse_inner_src(config: &mut ParsedConfig, flags: &LocalFlags, build_src_from_toml: &Option) { + config.src = if let Some(src) = flags.src.clone() { + src + } else if let Some(src) = build_src_from_toml.clone() { + src + } else { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + manifest_dir.parent().unwrap().parent().unwrap().to_owned() + }; +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs new file mode 100644 index 00000000..4f4fd6fc --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_stage0.rs @@ -0,0 +1,5 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +pub fn parse_inner_stage0(config: &mut ParsedConfig, toml: &LocalTomlConfig) { + // Removed build_helper dependency +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs new file mode 100644 index 00000000..b91da222 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parse_inner_toml.rs @@ -0,0 +1,36 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_flags::LocalFlags; +use crate::local_toml_config::LocalTomlConfig; +use std::path::Path; +use std::path::PathBuf; +use std::env; + +pub fn parse_inner_toml(config: &mut ParsedConfig, flags: &LocalFlags, get_toml: impl Fn(&Path) -> Result) -> LocalTomlConfig { + // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + let toml_path = flags + .config + .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = config.src.join(toml_path); + } + + // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + // but not if `config.toml` hasn't been created. + if !using_default_path || toml_path.exists() { + config.config = Some(if cfg!(not(feature = "bootstrap-self-test")) { + toml_path.canonicalize().unwrap() + } else { + toml_path.clone() + }); + get_toml(&toml_path).unwrap_or_else(|e| { + eprintln!("ERROR: Failed to parse '{}': {e}", toml_path.display()); + std::process::exit(2); + }) + } else { + config.config = None; + LocalTomlConfig::default() + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs new file mode 100644 index 00000000..9048bf3c --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/parsed_config.rs @@ -0,0 +1,173 @@ +// This will be the lib.rs for the new bootstrap-config-utils crate +use std::path::PathBuf; +use crate::target_selection::TargetSelection; +use crate::dry_run::DryRun; +use crate::local_ci_config::LocalCiConfig; + + + +#[derive(Debug, Default)] +pub struct ParsedConfig { + pub channel_file: Option, + pub version_file: Option, + pub tools_dir: Option, + pub llvm_project_dir: Option, + pub gcc_dir: Option, + pub config: Option, + // ... other fields ... + pub change_id: Option, + pub jobs: Option, + pub build_triple: Option, + pub out: PathBuf, + pub build: TargetSelection, + pub bypass_bootstrap_lock: bool, + pub llvm_optimize: Option, + pub ninja_in_file: Option, + pub llvm_static_stdcpp: Option, + pub llvm_libzstd: Option, + pub backtrace: Option, + pub rust_optimize_tests: Option, + pub docs: Option, + pub docs_minification: Option, + pub rust_rpath: Option, + pub rust_strip: Option, + pub rust_dist_src: Option, + pub deny_warnings: Option, + pub dist_include_mingw_linker: Option, + pub out_dir: Option, + pub initial_cargo_clippy: Option, + pub initial_rustc: Option, + pub initial_cargo: Option, + pub dry_run: DryRun, + pub hosts: Vec, + pub channel: Option, + pub codegen_tests: Option, + pub stdout_is_tty: Option, + pub stderr_is_tty: Option, + pub src: PathBuf, + pub ci: Option, + pub targets: Vec, + + pub nodejs: Option, + pub npm: Option, + pub gdb: Option, + pub lldb: Option, + pub python: Option, + pub reuse: Option, + pub submodules: Option, + pub android_ndk: Option, + pub bootstrap_cache_path: Option, + pub low_priority: Option, + pub compiler_docs: Option, + pub library_docs_private_items: Option, + pub locked_deps: Option, + pub vendor: Option, + pub full_bootstrap: Option, + pub extended: Option, + pub tools: Option>, + pub verbose: Option, + pub sanitizers: Option, + pub profiler: Option, + pub cargo_native_static: Option, + pub configure_args: Option>, + pub local_rebuild: Option, + pub print_step_timings: Option, + pub print_step_rusage: Option, + pub patch_binaries_for_nix: Option, + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: Option, + pub libdir: Option, + pub mandir: Option, + pub llvm_tests: bool, + pub llvm_offload: bool, + pub llvm_plugins: bool, + pub verbose_tests_flag: Option, + pub llvm_assertions: Option, + pub llvm_enzyme_flag: Option, + pub rust_optimize: Option, // Will be converted to RustOptimize enum later + pub omit_git_hash: bool, + pub rust_new_symbol_mangling: Option, + pub rust_frame_pointers: Option, + pub rust_stack_protector: Option, + pub jemalloc: Option, + pub test_compare_mode: Option, + pub description: Option, + pub incremental: bool, + pub lld_mode: Option, // Will be converted to LldMode enum later + pub llvm_bitcode_linker_enabled: Option, + pub rust_randomize_layout: bool, + pub llvm_tools_enabled: bool, + pub rustc_default_linker: Option, + pub musl_root: Option, + pub save_toolstates: Option, + pub backtrace_on_ice: Option, + pub rust_verify_llvm_ir: Option, + pub rust_thin_lto_import_instr_limit: Option, + pub rust_remap_debuginfo: Option, + pub control_flow_guard: Option, + pub ehcont_guard: Option, + pub llvm_libunwind_default: Option, + pub rust_codegen_backends: Vec, + pub rust_codegen_units: Option, + pub rust_codegen_units_std: Option, + pub rust_profile_use: Option, + pub rust_profile_generate: Option, + pub rust_lto: Option, // Will be converted to RustcLto enum later + pub rust_validate_mir_opts: Option, + pub reproducible_artifacts: bool, + pub download_rustc_commit: Option, + pub llvm_from_ci: bool, + pub llvm_thin_lto: Option, + pub llvm_release_debuginfo: Option, + pub llvm_link_shared: Option, + pub llvm_targets: Vec, + pub llvm_experimental_targets: Vec, + pub llvm_link_jobs: Option, + pub llvm_version_suffix: Option, + pub llvm_clang_cl: Option, + pub llvm_enable_projects: Vec, + pub llvm_cflags: Option, + pub llvm_cxxflags: Option, + pub llvm_ldflags: Option, + pub llvm_use_libcxx: Option, + pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_enable_warnings: bool, + pub ccache: Option, + pub llvm_build_config: Option, + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_compression_formats: Option>, + pub dist_compression_profile: Option, + pub dist_vendor: bool, + pub initial_rustfmt: Option, // Will be converted to RustfmtState enum later + pub lld_enabled: bool, + pub rust_std_features: std::collections::BTreeSet, + pub rustc_debug_assertions: bool, + pub std_debug_assertions: bool, + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: Option, // Will be converted to DebuginfoLevel enum later + pub rust_debuginfo_level_std: Option, // Will be converted to DebuginfoLevel enum later + pub rust_debuginfo_level_tools: Option, // Will be converted to DebuginfoLevel enum later + pub rust_debuginfo_level_tests: Option, // Will be converted to DebuginfoLevel enum later + pub optimized_compiler_builtins: bool, + pub compiletest_diff_tool: Option, + pub stage: usize, + pub cmd: Option, // Will be converted to Subcommand enum later + + // Nix-related fields + pub nixpkgs_path: Option, + pub rust_overlay_path: Option, + pub rust_bootstrap_nix_path: Option, + pub configuration_nix_path: Option, + pub rust_src_flake_path: Option, + pub rust_bootstrap_nix_flake_ref: Option, + pub rust_src_flake_ref: Option, +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs b/standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs new file mode 100644 index 00000000..4032a177 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/rust_channel_git_hash_config.rs @@ -0,0 +1,20 @@ +use crate::parsed_config::ParsedConfig; +use crate::local_toml_config::LocalTomlConfig; +use crate::config_applicator::ConfigApplicator; + +pub struct RustChannelGitHashConfigApplicator; + +impl ConfigApplicator for RustChannelGitHashConfigApplicator { + fn apply_to_config(&self, config: &mut ParsedConfig, toml: &LocalTomlConfig) { + let _is_user_configured_rust_channel = + if let Some(channel) = toml.rust.as_ref().and_then(|r| r.channel.clone()) { + config.channel = Some(channel); + true + } else { + false + }; + + config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(config.channel.as_deref() == Some("dev")); + // GitInfo assignments will be handled by the processor crate + } +} diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/target_selection.rs b/standalonex/src/bootstrap/src/core/config_utils/src/target_selection.rs new file mode 100644 index 00000000..63450e24 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/target_selection.rs @@ -0,0 +1,16 @@ +#[derive(Debug)] +pub struct TargetSelection(pub String); + + +impl TargetSelection { + pub fn from_user(s: &str) -> Self { + TargetSelection(s.to_string()) + } +} + +impl Default for TargetSelection { + fn default() -> Self { + TargetSelection::from_user("x86_64-unknown-linux-gnu") // Placeholder default + } +} + diff --git a/standalonex/src/bootstrap/src/core/config_utils/src/try_run.rs b/standalonex/src/bootstrap/src/core/config_utils/src/try_run.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/config_utils/src/try_run.rs @@ -0,0 +1 @@ + diff --git a/standalonex/src/bootstrap/src/core/download.rs b/standalonex/src/bootstrap/src/core/download.rs index 4ec5d70d..5903d886 100644 --- a/standalonex/src/bootstrap/src/core/download.rs +++ b/standalonex/src/bootstrap/src/core/download.rs @@ -47,14 +47,14 @@ impl Config { } pub(crate) fn create(&self, path: &Path, s: &str) { - if self.dry_run() { + if self.dry_run { return; } t!(fs::write(path, s)); } pub(crate) fn remove(&self, f: &Path) { - if self.dry_run() { + if self.dry_run { return; } fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); @@ -74,7 +74,7 @@ impl Config { /// Returns false if do not execute at all, otherwise returns its /// `status.success()`. pub(crate) fn check_run(&self, cmd: &mut BootstrapCommand) -> bool { - if self.dry_run() && !cmd.run_always { + if self.dry_run && !cmd.run_always { return true; } self.verbose(|| println!("running: {cmd:?}")); @@ -367,7 +367,7 @@ impl Config { self.verbose(|| println!("verifying {}", path.display())); - if self.dry_run() { + if self.dry_run { return false; } @@ -387,7 +387,7 @@ impl Config { reader.consume(l); } - let checksum = hex_encode(hasher.finalize().as_slice()); + let checksum = hex_encode(hasher.finalize()); let verified = checksum == expected; if !verified { @@ -508,7 +508,7 @@ impl Config { fn ci_component_contents(&self, stamp_file: &str) -> Vec { assert!(self.download_rustc()); - if self.dry_run() { + if self.dry_run { return vec![]; } @@ -628,7 +628,7 @@ impl Config { key: &str, destination: &str, ) { - if self.dry_run() { + if self.dry_run { return; } @@ -733,7 +733,7 @@ download-rustc = false let llvm_stamp = llvm_root.join(".llvm-stamp"); let llvm_sha = detect_llvm_sha(self, self.rust_info.is_managed_git_subrepository()); let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run { self.download_ci_llvm(&llvm_sha); if self.should_fix_bins_and_dylibs() { diff --git a/standalonex/src/bootstrap/src/core/generate_steps/git_modules/create_branch.rs b/standalonex/src/bootstrap/src/core/generate_steps/git_modules/create_branch.rs new file mode 100644 index 00000000..0c6ee7a9 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/generate_steps/git_modules/create_branch.rs @@ -0,0 +1,70 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub fn run_git_command( + current_dir: &Path, + args: &[&str], + error_message: &str, + dry_run: bool, +) -> Result<(), Box> { + println!("Running git command in CWD: {:?}", current_dir); + let command_str = format!("git {}", args.join(" ")); + if dry_run { + println!("Dry run: Would execute: {}", command_str); + return Ok(()); + } + println!("Executing: {}", command_str); + + let output = Command::new("git") + .current_dir(current_dir) + .args(args) + .output()?; + + if !output.status.success() { + eprintln!("Git command failed: {}", error_message); + eprintln!("Stdout: {}", String::from_utf8_lossy(&output.stdout)); + eprintln!("Stderr: {}", String::from_utf8_lossy(&output.stderr)); + return Err(error_message.into()); + } + Ok(()) +} + +pub fn create_and_push_branch( + repo_root: &PathBuf, + branch_name: &str, + base_branch_name: &str, + output_dir: &PathBuf, + dry_run: bool, +) -> Result<(), Box> { + println!("Performing Git operations..."); + + // Explicitly checkout the base branch to ensure a stable HEAD + run_git_command(repo_root, &["checkout", base_branch_name], "Failed to checkout base branch", dry_run)?; + + // Check if branch already exists + let branch_exists_output = Command::new("git") + .current_dir(repo_root) + .args(&["rev-parse", "--verify", branch_name]) + .output()?; + + if branch_exists_output.status.success() { + println!("Branch '{}' already exists. Checking it out.", branch_name); + run_git_command(repo_root, &["checkout", branch_name], "Failed to checkout existing branch", dry_run)?; + } else { + println!("Branch '{}' does not exist. Creating and checking it out.", branch_name); + run_git_command(repo_root, &["checkout", "-b", branch_name], "Failed to create and checkout new branch", dry_run)?; + } + + // Add generated files + run_git_command(repo_root, &["add", &output_dir.to_string_lossy()], "Failed to add generated files", dry_run)?; + + // Commit changes + let commit_message = format!("feat: Generated seed flake {}", branch_name); + run_git_command(repo_root, &["commit", "-m", &commit_message], "Failed to commit changes", dry_run)?; + + // Push branch + run_git_command(repo_root, &["push", "origin", branch_name], "Failed to push branch", dry_run)?; + println!("Successfully pushed branch: {}", branch_name); + + Ok(()) +} diff --git a/standalonex/src/bootstrap/src/core/mod.rs b/standalonex/src/bootstrap/src/core/mod.rs index 9e18d670..2a7e861f 100644 --- a/standalonex/src/bootstrap/src/core/mod.rs +++ b/standalonex/src/bootstrap/src/core/mod.rs @@ -4,3 +4,5 @@ pub(crate) mod config; pub(crate) mod download; pub(crate) mod metadata; pub(crate) mod sanity; +pub(crate) mod types; +pub(crate) mod nix_steps; diff --git a/standalonex/src/bootstrap/src/core/nix_steps/mod.rs b/standalonex/src/bootstrap/src/core/nix_steps/mod.rs new file mode 100644 index 00000000..9ec0d42d --- /dev/null +++ b/standalonex/src/bootstrap/src/core/nix_steps/mod.rs @@ -0,0 +1 @@ +pub mod nix_build; \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/nix_steps/nix_build.rs b/standalonex/src/bootstrap/src/core/nix_steps/nix_build.rs new file mode 100644 index 00000000..d7b6ea5e --- /dev/null +++ b/standalonex/src/bootstrap/src/core/nix_steps/nix_build.rs @@ -0,0 +1,20 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub fn run_nix_build(flake_dir: &Path) -> Result<(), Box> { + println!("Running Nix build on generated flake..."); + let nix_build_output = Command::new("nix") + .arg("build") + .arg(".#default") // Use .#default when current_dir is the flake directory + .current_dir(flake_dir) // Run nix build from the generated flake directory + .output()?; + + if !nix_build_output.status.success() { + eprintln!("Nix build failed!"); + eprintln!("Stdout: {}", String::from_utf8_lossy(&nix_build_output.stdout)); + eprintln!("Stderr: {}", String::from_utf8_lossy(&nix_build_output.stderr)); + return Err("Nix build failed".into()); + } + println!("Nix build passed. Output in result link."); + Ok(()) +} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/core/sanity.rs b/standalonex/src/bootstrap/src/core/sanity.rs index fcc9ea86..f8989cd4 100644 --- a/standalonex/src/bootstrap/src/core/sanity.rs +++ b/standalonex/src/bootstrap/src/core/sanity.rs @@ -114,7 +114,7 @@ pub fn check(build: &mut Build) { // Ensure that a compatible version of libstdc++ is available on the system when using `llvm.download-ci-llvm`. #[cfg(not(feature = "bootstrap-self-test"))] - if !build.config.dry_run() && !build.build.is_msvc() && build.config.llvm_from_ci { + if !build.config.dry_run && !build.build.is_msvc() && build.config.llvm_from_ci { let builder = Builder::new(build); let libcxx_version = builder.ensure(tool::LibcxxVersionTool { target: build.build }); @@ -292,7 +292,7 @@ than building it. // sbf target relies on in-tree built llvm, // which doesn't exist when this check runs - if !build.config.dry_run() && !target.contains("sbf") && !target.contains("bpf") { + if !build.config.dry_run && !target.contains("sbf") && !target.contains("bpf") { cmd_finder.must_have(build.cc(*target)); if let Some(ar) = build.ar(*target) { cmd_finder.must_have(ar); @@ -300,7 +300,7 @@ than building it. } } - if !build.config.dry_run() { + if !build.config.dry_run { for host in &build.hosts { cmd_finder.must_have(build.cxx(*host).unwrap()); diff --git a/standalonex/src/bootstrap/src/core/types.rs b/standalonex/src/bootstrap/src/core/types.rs new file mode 100644 index 00000000..bffdecb2 --- /dev/null +++ b/standalonex/src/bootstrap/src/core/types.rs @@ -0,0 +1,156 @@ +use std::fmt::Debug; +use std::hash::Hash; + +use crate::builder::Builder; +use crate::{Subcommand, TargetSelection, Kind, Compiler}; + +pub trait RustcTaskConfig: Sized + Debug + Clone + PartialEq + Eq + Hash { + fn default_config(builder: &Builder<'_>) -> Self; +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub compiler: Compiler, + pub target: TargetSelection, + pub crates: Vec, + pub config: C, // Task-specific configuration +} + +pub trait StdTaskConfig: Sized + Debug + Clone + PartialEq + Eq + Hash { + fn get_crates(&self) -> &Vec; + fn get_override_build_kind(&self) -> Option; + fn default_config(builder: &Builder<'_>) -> Self; +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Std { + pub target: TargetSelection, + pub config: C, + pub crates: Vec, +} + +// Concrete implementations of StdTaskConfig +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CheckStdConfig { + pub override_build_kind: Option, +} + +impl CheckStdConfig { + pub fn new(override_build_kind: Option) -> Self { + Self { override_build_kind } + } +} + +impl StdTaskConfig for CheckStdConfig { + fn get_crates(&self) -> &Vec { + &vec![] + } + fn get_override_build_kind(&self) -> Option { + self.override_build_kind + } + fn default_config(_builder: &Builder<'_>) -> Self { + Self::new(None) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ClippyStdConfig { + pub config: LintConfig, +} + +impl ClippyStdConfig { + pub fn new(config: LintConfig) -> Self { + Self { config } + } +} + +impl StdTaskConfig for ClippyStdConfig { + fn get_crates(&self) -> &Vec { + &vec![] + } + fn get_override_build_kind(&self) -> Option { + None + } + fn default_config(builder: &Builder<'_>) -> Self { + Self::new(LintConfig::new(builder)) + } +} + + + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CheckRustcConfig { + pub override_build_kind: Option, +} + +impl CheckRustcConfig { + pub fn new(override_build_kind: Option) -> Self { + Self { override_build_kind } + } +} + +impl RustcTaskConfig for CheckRustcConfig { + fn default_config(_builder: &Builder<'_>) -> Self { + Self::new(None) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LintConfig { + pub allow: Vec, + pub warn: Vec, + pub deny: Vec, + pub forbid: Vec, +} + +impl LintConfig { + pub fn new(builder: &Builder<'_>) -> Self { + match builder.config.cmd.clone() { + Subcommand::Clippy { allow, deny, warn, forbid, .. } => { + Self { allow, warn, deny, forbid } + } + _ => unreachable!("LintConfig can only be called from `clippy` subcommands."), + } + } + + pub fn merge(&self, other: &Self) -> Self { + let merged = |self_attr: &[String], other_attr: &[String]| -> Vec { + self_attr.iter().cloned().chain(other_attr.iter().cloned()).collect() + }; + // This is written this way to ensure we get a compiler error if we add a new field. + Self { + allow: merged(&self.allow, &other.allow), + warn: merged(&self.warn, &other.warn), + deny: merged(&self.deny, &other.deny), + forbid: merged(&self.forbid, &other.forbid), + } + } +} + +impl RustcTaskConfig for LintConfig { + fn default_config(builder: &Builder<'_>) -> Self { + Self::new(builder) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CompileRustcConfig { + // No specific fields needed if compiler, target, crates are in generic Rustc +} + +impl RustcTaskConfig for CompileRustcConfig {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct DistRustcConfig { + // No specific fields needed if compiler is in generic Rustc +} + +impl RustcTaskConfig for DistRustcConfig {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct DocRustcConfig { + pub stage: u32, + pub validate: bool, +} + +impl RustcTaskConfig for DocRustcConfig {} \ No newline at end of file diff --git a/standalonex/src/bootstrap/src/crate_struct.rs b/standalonex/src/bootstrap/src/crate_struct.rs new file mode 100644 index 00000000..1aee04df --- /dev/null +++ b/standalonex/src/bootstrap/src/crate_struct.rs @@ -0,0 +1,19 @@ +use std::collections::HashSet; +use std::path::PathBuf; + +use crate::Build; + +#[derive(Debug, Clone)] +pub struct Crate { + pub name: String, + pub deps: HashSet, + pub path: PathBuf, + pub has_lib: bool, + pub features: Vec, +} + +impl Crate { + pub fn local_path(&self, build: &Build) -> PathBuf { + self.path.strip_prefix(&build.config.src).unwrap().into() + } +} diff --git a/standalonex/src/bootstrap/src/dependency_type.rs b/standalonex/src/bootstrap/src/dependency_type.rs new file mode 100644 index 00000000..29f1dc6c --- /dev/null +++ b/standalonex/src/bootstrap/src/dependency_type.rs @@ -0,0 +1,10 @@ +/// When building Rust various objects are handled differently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DependencyType { + /// Libraries originating from proc-macros. + Host, + /// Typical Rust libraries. + Target, + /// Non Rust libraries and objects shipped to ease usage of certain targets. + TargetSelfContained, +} diff --git a/standalonex/src/bootstrap/src/enums.rs b/standalonex/src/bootstrap/src/enums.rs new file mode 100644 index 00000000..954192ab --- /dev/null +++ b/standalonex/src/bootstrap/src/enums.rs @@ -0,0 +1,65 @@ +use crate::Mode; + +pub enum DocTests { + /// Run normal tests and doc tests (default). + Yes, + /// Do not run any doc tests. + No, + /// Only run doc tests. + Only, +} + +pub enum GitRepo { + Rustc, + Llvm, +} + +pub enum CLang { + C, + Cxx, +} + +/// The various "modes" of invoking Cargo. +/// +/// These entries currently correspond to the various output directories of the +/// build system, with each mod generating output in a different directory. +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Mode { + /// Build the standard library, placing output in the "stageN-std" directory. + Std, + + /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. + Rustc, + + /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. + Codegen, + + /// Build a tool, placing output in the "stage0-bootstrap-tools" + /// directory. This is for miscellaneous sets of tools that are built + /// using the bootstrap stage0 compiler in its entirety (target libraries + /// and all). Typically these tools compile with stable Rust. + /// + /// Only works for stage 0. + ToolBootstrap, + + /// Build a tool which uses the locally built std, placing output in the + /// "stageN-tools" directory. Its usage is quite rare, mainly used by + /// compiletest which needs libtest. + ToolStd, + + /// Build a tool which uses the locally built rustc and the target std, + /// placing the output in the "stageN-tools" directory. This is used for + /// anything that needs a fully functional rustc, such as rustdoc, clippy, + /// cargo, rls, rustfmt, miri, etc. + ToolRustc, +} + +impl Mode { + pub fn is_tool(&self) -> bool { + matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) + } + + pub fn must_support_dlopen(&self) -> bool { + matches!(self, Mode::Std | Mode::Codegen) + } +} diff --git a/standalonex/src/bootstrap/src/helpers.rs b/standalonex/src/bootstrap/src/helpers.rs new file mode 100644 index 00000000..97f18cfb --- /dev/null +++ b/standalonex/src/bootstrap/src/helpers.rs @@ -0,0 +1,25 @@ +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::io; + +use crate::Build; +use crate::flags; +use termcolor::{ColorChoice, StandardStream, WriteColor}; + +pub fn envify(s: &str) -> String { + s.chars() + .map(|c| match c { + '-' => '_', + c => c, + }) + .flat_map(|c| c.to_uppercase()) + .collect() +} + +#[cfg(unix)] +pub fn chmod(path: &Path, perms: u32) { + use std::os::unix::fs::*; + t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); +} +#[cfg(windows)] +pub fn chmod(_path: &Path, _perms: u32) {} diff --git a/standalonex/src/bootstrap/src/lib.rs b/standalonex/src/bootstrap/src/lib.rs index fe0d4f26..9cf0cb59 100644 --- a/standalonex/src/bootstrap/src/lib.rs +++ b/standalonex/src/bootstrap/src/lib.rs @@ -1,20 +1,21 @@ -//! Implementation of bootstrap, the Rust build system. -//! -//! This module, and its descendants, are the implementation of the Rust build -//! system. Most of this build system is backed by Cargo but the outer layer -//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. The goals of bootstrap are: -//! -//! * To be an easily understandable, easily extensible, and maintainable build -//! system. -//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka -//! crates.io and Cargo. -//! * A standard interface to build across all platforms, including MSVC -//! -//! ## Further information -//! -//! More documentation can be found in each respective module below, and you can -//! also check out the `src/bootstrap/README.md` file for more information. +use crate::prelude::*; +/// Implementation of bootstrap, the Rust build system. +/// +/// This module, and its descendants, are the implementation of the Rust build +/// system. Most of this build system is backed by Cargo but the outer layer +/// here serves as the ability to orchestrate calling Cargo, sequencing Cargo +/// builds, building artifacts like LLVM, etc. The goals of bootstrap are: +/// +/// * To be an easily understandable, easily extensible, and maintainable build +/// system. +/// * Leverage standard tools in the Rust ecosystem to build the compiler, aka +/// crates.io and Cargo. +/// * A standard interface to build across all platforms, including MSVC +/// +/// ## Further information +/// +/// More documentation can be found in each respective module below, and you can +/// also check out the `src/bootstrap/README.md` file for more information. use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap, HashSet}; @@ -33,9 +34,17 @@ use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::channel::GitInfo; use utils::helpers::hex_encode; + use crate::core::builder; use crate::core::builder::{Builder, Kind}; -use crate::core::config::{DryRun, LldMode, LlvmLibunwind, Target, TargetSelection, flags}; + +pub use crate::core::config::*; +use crate::core::config::dry_run::DryRun; +use crate::core::config::lld_mode::LldMode; +use crate::core::config::llvm_lib_unwind::LlvmLibunwind; +use crate::core::config::target_selection::Target; +use crate::core::config::target_selection::TargetSelection; +use crate::core::config::flags; use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; use crate::utils::helpers::{ self, dir_is_empty, exe, libdir, mtime, output, set_file_times, symlink_dir, @@ -43,14 +52,25 @@ use crate::utils::helpers::{ mod core; mod utils; +pub mod prelude; pub use core::builder::PathSet; pub use core::config::Config; -pub use core::config::flags::{Flags, Subcommand}; +pub use core::config::flags::Flags; +pub use crate::Subcommand; -pub use utils::change_tracker::{ - CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes, -}; +pub use utils::change_tracker::{CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes,}; + +macro_rules! forward { + ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { + impl Build { + $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { + self.config.$fn( $($param),* ) + } )+ + } + } +} +pub(crate) use forward; const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report @@ -274,6 +294,22 @@ forward! { llvm_link_shared() -> bool, download_rustc() -> bool, initial_rustfmt() -> Option, + last_modified_commit(modified_paths: &[&str], option_name: &str, if_unchanged: bool) -> Option, + needs_sanitizer_runtime_built(target: TargetSelection) -> bool, + llvm_libunwind(target: TargetSelection) -> LlvmLibunwind, + ci_llvm_root() -> PathBuf, + profiler_path(target: TargetSelection) -> Option<&str>, + profiler_enabled(target: TargetSelection) -> bool, + ci_rustc_dir() -> PathBuf, + default_codegen_backend(target: TargetSelection) -> Option, + libdir_relative() -> Option<&Path>, + llvm_enabled(target: TargetSelection) -> bool, + codegen_backends(target: TargetSelection) -> &[String], + git_config() -> GitConfig<'_>, + update_submodule(relative_path: &str), + submodules() -> bool, + args() -> Vec<&str>, + test_args() -> Vec<&str>, } impl Build { @@ -312,7 +348,7 @@ impl Build { let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); - let initial_target_libdir_str = if config.dry_run() { + let initial_target_libdir_str = if config.dry_run { "/dummy/lib/path/to/lib/".to_string() } else { output( @@ -326,7 +362,7 @@ impl Build { let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); let initial_lld = initial_target_dir.join("bin").join("rust-lld"); - let initial_sysroot = if config.dry_run() { + let initial_sysroot = if config.dry_run { "/dummy".to_string() } else { output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) @@ -367,13 +403,13 @@ impl Build { if bootstrap_out.ends_with("deps") { bootstrap_out.pop(); } - if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { - // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!( - "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", - bootstrap_out.display() - ) - } + // if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { + // // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + // panic!( + // "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", + // bootstrap_out.display() + // ) + // } if rust_info.is_from_tarball() && config.description.is_none() { config.description = Some("built from a source tarball".to_owned()); @@ -582,12 +618,12 @@ impl Build { return core::build_steps::format::format( &builder::Builder::new(self), *check, - *all, + all, &self.config.paths, ); } Subcommand::Suggest { run } => { - return core::build_steps::suggest::suggest(&builder::Builder::new(self), *run); + return core::build_steps::suggest::suggest(&builder::Builder::new(self), run); } Subcommand::Perf { .. } => { return core::build_steps::perf::perf(&builder::Builder::new(self)); @@ -595,7 +631,7 @@ impl Build { _ => (), } - if !self.config.dry_run() { + if !self.config.dry_run { { // We first do a dry-run. This is a sanity-check to ensure that // steps don't do anything expensive in the dry-run. @@ -920,7 +956,7 @@ impl Build { stderr: OutputMode, ) -> CommandOutput { command.mark_as_executed(); - if self.config.dry_run() && !command.run_always { + if self.config.dry_run && !command.run_always { return CommandOutput::default(); } @@ -1192,7 +1228,7 @@ Executed at: {executed_at}"#, /// Returns the path to the C compiler for the target specified. fn cc(&self, target: TargetSelection) -> PathBuf { - if self.config.dry_run() { + if self.config.dry_run { return PathBuf::new(); } self.cc.borrow()[&target].path().into() @@ -1201,7 +1237,7 @@ Executed at: {executed_at}"#, /// Returns a list of flags to pass to the C compiler for the target /// specified. fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { - if self.config.dry_run() { + if self.config.dry_run { return Vec::new(); } let base = match c { @@ -1247,7 +1283,7 @@ Executed at: {executed_at}"#, /// Returns the path to the `ar` archive utility for the target specified. fn ar(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { + if self.config.dry_run { return None; } self.ar.borrow().get(&target).cloned() @@ -1255,7 +1291,7 @@ Executed at: {executed_at}"#, /// Returns the path to the `ranlib` utility for the target specified. fn ranlib(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { + if self.config.dry_run { return None; } self.ranlib.borrow().get(&target).cloned() @@ -1263,7 +1299,7 @@ Executed at: {executed_at}"#, /// Returns the path to the C++ compiler for the target specified. fn cxx(&self, target: TargetSelection) -> Result { - if self.config.dry_run() { + if self.config.dry_run { return Ok(PathBuf::new()); } match self.cxx.borrow().get(&target) { @@ -1274,7 +1310,7 @@ Executed at: {executed_at}"#, /// Returns the path to the linker for the given target if it needs to be overridden. fn linker(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { + if self.config.dry_run { return Some(PathBuf::new()); } if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) @@ -1660,7 +1696,7 @@ Executed at: {executed_at}"#, } fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { - if self.config.dry_run() { + if self.config.dry_run { return Vec::new(); } @@ -1709,7 +1745,7 @@ Executed at: {executed_at}"#, } fn copy_link_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { - if self.config.dry_run() { + if self.config.dry_run { return; } self.verbose_than(1, || println!("Copy/Link {src:?} to {dst:?}")); @@ -1736,7 +1772,7 @@ Executed at: {executed_at}"#, } } if let Ok(()) = fs::hard_link(&src, dst) { - // Attempt to "easy copy" by creating a hard link (symlinks are priviledged on windows), + // Attempt to "easy copy" by creating a hard link (symlinks are privileged on windows), // but if that fails just fall back to a slow `copy` operation. } else { if let Err(e) = fs::copy(&src, dst) { @@ -1757,7 +1793,7 @@ Executed at: {executed_at}"#, /// when this function is called. /// Will attempt to use hard links if possible and fall back to copying. pub fn cp_link_r(&self, src: &Path, dst: &Path) { - if self.config.dry_run() { + if self.config.dry_run { return; } for f in self.read_dir(src) { @@ -1817,7 +1853,7 @@ Executed at: {executed_at}"#, } fn install(&self, src: &Path, dstdir: &Path, perms: u32) { - if self.config.dry_run() { + if self.config.dry_run { return; } let dst = dstdir.join(src.file_name().unwrap()); @@ -1831,21 +1867,21 @@ Executed at: {executed_at}"#, } fn read(&self, path: &Path) -> String { - if self.config.dry_run() { + if self.config.dry_run { return String::new(); } t!(fs::read_to_string(path)) } fn create_dir(&self, dir: &Path) { - if self.config.dry_run() { + if self.config.dry_run { return; } t!(fs::create_dir_all(dir)) } fn remove_dir(&self, dir: &Path) { - if self.config.dry_run() { + if self.config.dry_run { return; } t!(fs::remove_dir_all(dir)) @@ -1854,18 +1890,21 @@ Executed at: {executed_at}"#, fn read_dir(&self, dir: &Path) -> impl Iterator { let iter = match fs::read_dir(dir) { Ok(v) => v, - Err(_) if self.config.dry_run() => return vec![].into_iter(), + Err(_) if self.config.dry_run => return vec![].into_iter(), Err(err) => panic!("could not read dir {dir:?}: {err:?}"), }; iter.map(|e| t!(e)).collect::>().into_iter() } fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { - #[cfg(unix)] - use std::os::unix::fs::symlink as symlink_file; - #[cfg(windows)] - use std::os::windows::fs::symlink_file; - if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } + if self.config.dry_run { return Ok(()); } + if cfg!(unix) { + std::os::unix::fs::symlink(src.as_ref(), link.as_ref()) + } /* else if cfg!(windows) { + std::os::windows::fs::symlink_file(src.as_ref(), link.as_ref()) + } */ else { + Err(io::Error::new(io::ErrorKind::Other, "symlinks not supported on this platform")) + } } /// Returns if config.ninja is enabled, and checks for ninja existence, @@ -1911,30 +1950,7 @@ to download LLVM rather than building it. self.config.ninja_in_file } - pub fn colored_stdout R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) - } - - pub fn colored_stderr R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) - } - fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R - where - C: Fn(ColorChoice) -> StandardStream, - F: FnOnce(&mut dyn WriteColor) -> R, - { - let choice = match self.config.color { - flags::Color::Always => ColorChoice::Always, - flags::Color::Never => ColorChoice::Never, - flags::Color::Auto if !is_tty => ColorChoice::Never, - flags::Color::Auto => ColorChoice::Auto, - }; - let mut stream = constructor(choice); - let result = f(&mut stream); - stream.reset().unwrap(); - result - } } #[cfg(unix)] @@ -2010,7 +2026,7 @@ pub fn generate_smart_stamp_hash( hasher.update(status); hasher.update(additional_input); - hex_encode(hasher.finalize().as_slice()) + hex_encode(hasher.finalize()) } /// Ensures that the behavior dump directory is properly initialized. diff --git a/standalonex/src/bootstrap/src/prelude.rs b/standalonex/src/bootstrap/src/prelude.rs new file mode 100644 index 00000000..8a41927c --- /dev/null +++ b/standalonex/src/bootstrap/src/prelude.rs @@ -0,0 +1,36 @@ +pub use std::path::{Path, PathBuf}; +pub use std::collections::{HashMap, HashSet, BTreeSet}; +pub use std::cell::{Cell, RefCell}; +pub use std::fmt::{self, Display}; +pub use std::str::FromStr; +pub use std::process::Command; +pub use std::env; +pub use std::cmp; +pub use std::sync::OnceLock; + +pub use serde::{Deserialize, Serialize}; +pub use clap::{ValueEnum, CommandFactory, Parser, Subcommand}; + +pub use build_helper::exit; +pub use crate::utils::helpers::t; + +// Modules from src/core/build_steps +pub use crate::core::build_steps::vendor; +pub use crate::core::build_steps::tool; +pub use crate::core::build_steps::toolstate; +pub use crate::core::build_steps::test; +pub use crate::core::build_steps::synthetic_targets; +pub use crate::core::build_steps::suggest; +pub use crate::core::build_steps::setup; +pub use crate::core::build_steps::run; +pub use crate::core::build_steps::perf; +pub use crate::core::build_steps::llvm; +pub use crate::core::build_steps::install; +pub use crate::core::build_steps::gcc; +pub use crate::core::build_steps::format; +pub use crate::core::build_steps::doc; +pub use crate::core::build_steps::dist; +pub use crate::core::build_steps::compile; +pub use crate::core::build_steps::clippy; +pub use crate::core::build_steps::clean; +pub use crate::core::build_steps::check; diff --git a/standalonex/src/bootstrap/src/utils/helpers.rs b/standalonex/src/bootstrap/src/utils/helpers.rs index c226d07d..bb91039b 100644 --- a/standalonex/src/bootstrap/src/utils/helpers.rs +++ b/standalonex/src/bootstrap/src/utils/helpers.rs @@ -128,7 +128,7 @@ pub struct TimeIt(bool, Instant); /// Returns an RAII structure that prints out how long it took to drop. pub fn timeit(builder: &Builder<'_>) -> TimeIt { - TimeIt(builder.config.dry_run(), Instant::now()) + TimeIt(builder.config.dry_run, Instant::now()) } impl Drop for TimeIt { @@ -151,7 +151,7 @@ pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { /// Symlinks two directories, using junctions on Windows and normal symlinks on /// Unix. pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { - if config.dry_run() { + if config.dry_run { return Ok(()); } let _ = fs::remove_dir_all(link); diff --git a/standalonex/src/bootstrap/src/utils/metrics.rs b/standalonex/src/bootstrap/src/utils/metrics.rs index 3b31fa36..346580b9 100644 --- a/standalonex/src/bootstrap/src/utils/metrics.rs +++ b/standalonex/src/bootstrap/src/utils/metrics.rs @@ -68,7 +68,7 @@ impl BuildMetrics { pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -96,7 +96,7 @@ impl BuildMetrics { pub(crate) fn exit_step(&self, builder: &Builder<'_>) { // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -119,7 +119,7 @@ impl BuildMetrics { pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { + if builder.config.dry_run { return; } @@ -130,7 +130,7 @@ impl BuildMetrics { pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { + if builder.config.dry_run { return; } diff --git a/standalonex/src/bootstrap/src/utils/render_tests.rs b/standalonex/src/bootstrap/src/utils/render_tests.rs index eb2c8254..68d7c884 100644 --- a/standalonex/src/bootstrap/src/utils/render_tests.rs +++ b/standalonex/src/bootstrap/src/utils/render_tests.rs @@ -34,7 +34,7 @@ pub(crate) fn try_run_tests( cmd: &mut BootstrapCommand, stream: bool, ) -> bool { - if builder.config.dry_run() { + if builder.config.dry_run { cmd.mark_as_executed(); return true; } diff --git a/standalonex/src/bootstrap/src/utils/tarball.rs b/standalonex/src/bootstrap/src/utils/tarball.rs index 3c6c7a7f..8c750725 100644 --- a/standalonex/src/bootstrap/src/utils/tarball.rs +++ b/standalonex/src/bootstrap/src/utils/tarball.rs @@ -390,7 +390,7 @@ impl<'a> Tarball<'a> { // Ensure there are no symbolic links in the tarball. In particular, // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. let decompressed_output = self.temp_dir.join(&package_name); - if !self.builder.config.dry_run() && !self.permit_symlinks { + if !self.builder.config.dry_run && !self.permit_symlinks { for entry in walkdir::WalkDir::new(&decompressed_output) { let entry = t!(entry); if entry.path_is_symlink() { diff --git a/standalonex/src/bootstrap/src/version b/standalonex/src/bootstrap/src/version new file mode 100644 index 00000000..d944efab --- /dev/null +++ b/standalonex/src/bootstrap/src/version @@ -0,0 +1 @@ +1.84.1 \ No newline at end of file diff --git a/standalonex/src/bootstrap/stage0/config.toml b/standalonex/src/bootstrap/stage0/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/standalonex/src/bootstrap/stage0/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/standalonex/src/bootstrap/stage0/config.toml.old b/standalonex/src/bootstrap/stage0/config.toml.old new file mode 100644 index 00000000..2fc2c498 --- /dev/null +++ b/standalonex/src/bootstrap/stage0/config.toml.old @@ -0,0 +1,4 @@ +vendor = true +#patch-binaries-for-nix=true +#rustc = "/nix/store/qdz7ccxq8k902nyzc9pb606cp1pam12c-rust-default-1.84.1/bin/rustc" +#cargo = "/nix/store/qdz7ccxq8k902nyzc9pb606cp1pam12c-rust-default-1.84.1/bin/cargo" diff --git a/standalonex/src/bootstrap/stage1/config.toml b/standalonex/src/bootstrap/stage1/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/standalonex/src/bootstrap/stage1/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/standalonex/src/bootstrap/stage2/config.toml b/standalonex/src/bootstrap/stage2/config.toml new file mode 100644 index 00000000..557e2196 --- /dev/null +++ b/standalonex/src/bootstrap/stage2/config.toml @@ -0,0 +1 @@ +[rust]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[build]\nrustc = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc"\ncargo = "/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo"\n\n[env]\nHOME = "$TMPDIR/home"\nCARGO_HOME = "$TMPDIR/cargo-home"\n \ No newline at end of file diff --git a/standalonex/src/bootstrap/test.sh b/standalonex/src/bootstrap/test.sh new file mode 100755 index 00000000..92163dce --- /dev/null +++ b/standalonex/src/bootstrap/test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +pushd /data/data/com.termux.nix/files/home/rust-bootstrap-nix/standalonex/src/bootstrap +cargo run build --stage 0 +#RUST_BACKTRACE=1 ./target/debug/bootstrap check +popd diff --git a/standalonex/src/bootstrap_test.py b/standalonex/src/bootstrap_test.py deleted file mode 100644 index 70ed12b9..00000000 --- a/standalonex/src/bootstrap_test.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Bootstrap tests - -Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstrap_test.py`.""" - -from __future__ import absolute_import, division, print_function -import os -import unittest -from unittest.mock import patch -import tempfile -import hashlib -import sys - -from shutil import rmtree - -# Allow running this from the top-level directory. -bootstrap_dir = os.path.dirname(os.path.abspath(__file__)) -# For the import below, have Python search in src/bootstrap first. -sys.path.insert(0, bootstrap_dir) -import bootstrap # noqa: E402 -import configure # noqa: E402 - -def serialize_and_parse(configure_args, bootstrap_args=None): - from io import StringIO - - if bootstrap_args is None: - bootstrap_args = bootstrap.FakeArgs() - - section_order, sections, targets = configure.parse_args(configure_args) - buffer = StringIO() - configure.write_config_toml(buffer, section_order, targets, sections) - build = bootstrap.RustBuild(config_toml=buffer.getvalue(), args=bootstrap_args) - - try: - import tomllib - # Verify this is actually valid TOML. - tomllib.loads(build.config_toml) - except ImportError: - print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) - return build - - -class VerifyTestCase(unittest.TestCase): - """Test Case for verify""" - def setUp(self): - self.container = tempfile.mkdtemp() - self.src = os.path.join(self.container, "src.txt") - self.bad_src = os.path.join(self.container, "bad.txt") - content = "Hello world" - - self.expected = hashlib.sha256(content.encode("utf-8")).hexdigest() - - with open(self.src, "w") as src: - src.write(content) - with open(self.bad_src, "w") as bad: - bad.write("Hello!") - - def tearDown(self): - rmtree(self.container) - - def test_valid_file(self): - """Check if the sha256 sum of the given file is valid""" - self.assertTrue(bootstrap.verify(self.src, self.expected, False)) - - def test_invalid_file(self): - """Should verify that the file is invalid""" - self.assertFalse(bootstrap.verify(self.bad_src, self.expected, False)) - - -class ProgramOutOfDate(unittest.TestCase): - """Test if a program is out of date""" - def setUp(self): - self.container = tempfile.mkdtemp() - os.mkdir(os.path.join(self.container, "stage0")) - self.build = bootstrap.RustBuild() - self.build.date = "2017-06-15" - self.build.build_dir = self.container - self.rustc_stamp_path = os.path.join(self.container, "stage0", - ".rustc-stamp") - self.key = self.build.date + str(None) - - def tearDown(self): - rmtree(self.container) - - def test_stamp_path_does_not_exist(self): - """Return True when the stamp file does not exist""" - if os.path.exists(self.rustc_stamp_path): - os.unlink(self.rustc_stamp_path) - self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - def test_dates_are_different(self): - """Return True when the dates are different""" - with open(self.rustc_stamp_path, "w") as rustc_stamp: - rustc_stamp.write("2017-06-14None") - self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - def test_same_dates(self): - """Return False both dates match""" - with open(self.rustc_stamp_path, "w") as rustc_stamp: - rustc_stamp.write("2017-06-15None") - self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) - - -class ParseArgsInConfigure(unittest.TestCase): - """Test if `parse_args` function in `configure.py` works properly""" - @patch("configure.err") - def test_unknown_args(self, err): - # It should be print an error message if the argument doesn't start with '--' - configure.parse_args(["enable-full-tools"]) - err.assert_called_with("Option 'enable-full-tools' is not recognized") - err.reset_mock() - # It should be print an error message if the argument is not recognized - configure.parse_args(["--some-random-flag"]) - err.assert_called_with("Option '--some-random-flag' is not recognized") - - @patch("configure.err") - def test_need_value_args(self, err): - """It should print an error message if a required argument value is missing""" - configure.parse_args(["--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - - @patch("configure.err") - def test_option_checking(self, err): - # Options should be checked even if `--enable-option-checking` is not passed - configure.parse_args(["--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - err.reset_mock() - # Options should be checked if `--enable-option-checking` is passed - configure.parse_args(["--enable-option-checking", "--target"]) - err.assert_called_with("Option '--target' needs a value (--target=val)") - err.reset_mock() - # Options should not be checked if `--disable-option-checking` is passed - configure.parse_args(["--disable-option-checking", "--target"]) - err.assert_not_called() - - @patch("configure.parse_example_config", lambda known_args, _: known_args) - def test_known_args(self): - # It should contain known and correct arguments - known_args = configure.parse_args(["--enable-full-tools"]) - self.assertTrue(known_args["full-tools"][0][1]) - known_args = configure.parse_args(["--disable-full-tools"]) - self.assertFalse(known_args["full-tools"][0][1]) - # It should contain known arguments and their values - known_args = configure.parse_args(["--target=x86_64-unknown-linux-gnu"]) - self.assertEqual(known_args["target"][0][1], "x86_64-unknown-linux-gnu") - known_args = configure.parse_args(["--target", "x86_64-unknown-linux-gnu"]) - self.assertEqual(known_args["target"][0][1], "x86_64-unknown-linux-gnu") - - -class GenerateAndParseConfig(unittest.TestCase): - """Test that we can serialize and deserialize a config.toml file""" - def test_no_args(self): - build = serialize_and_parse([]) - self.assertEqual(build.get_toml("profile"), 'dist') - self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) - - def test_set_section(self): - build = serialize_and_parse(["--set", "llvm.download-ci-llvm"]) - self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), 'true') - - def test_set_target(self): - build = serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"]) - self.assertEqual(build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), 'gcc') - - def test_set_top_level(self): - build = serialize_and_parse(["--set", "profile=compiler"]) - self.assertEqual(build.get_toml("profile"), 'compiler') - - def test_set_codegen_backends(self): - build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift']"), -1) - build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift,llvm"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift', 'llvm']"), -1) - build = serialize_and_parse(["--enable-full-tools"]) - self.assertNotEqual(build.config_toml.find("codegen-backends = ['llvm']"), -1) - - -class BuildBootstrap(unittest.TestCase): - """Test that we generate the appropriate arguments when building bootstrap""" - - def build_args(self, configure_args=None, args=None, env=None): - if configure_args is None: - configure_args = [] - if args is None: - args = [] - if env is None: - env = {} - - # This test ends up invoking build_bootstrap_cmd, which searches for - # the Cargo binary and errors out if it cannot be found. This is not a - # problem in most cases, but there is a scenario where it would cause - # the test to fail. - # - # When a custom local Cargo is configured in config.toml (with the - # build.cargo setting), no Cargo is downloaded to any location known by - # bootstrap, and bootstrap relies on that setting to find it. - # - # In this test though we are not using the config.toml of the caller: - # we are generating a blank one instead. If we don't set build.cargo in - # it, the test will have no way to find Cargo, failing the test. - cargo_bin = os.environ.get("BOOTSTRAP_TEST_CARGO_BIN") - if cargo_bin is not None: - configure_args += ["--set", "build.cargo=" + cargo_bin] - rustc_bin = os.environ.get("BOOTSTRAP_TEST_RUSTC_BIN") - if rustc_bin is not None: - configure_args += ["--set", "build.rustc=" + rustc_bin] - - env = env.copy() - env["PATH"] = os.environ["PATH"] - - parsed = bootstrap.parse_args(args) - build = serialize_and_parse(configure_args, parsed) - # Make these optional so that `python -m unittest` works when run manually. - build_dir = os.environ.get("BUILD_DIR") - if build_dir is not None: - build.build_dir = build_dir - build_platform = os.environ.get("BUILD_PLATFORM") - if build_platform is not None: - build.build = build_platform - return build.build_bootstrap_cmd(env), env - - def test_cargoflags(self): - args, _ = self.build_args(env={"CARGOFLAGS": "--timings"}) - self.assertTrue("--timings" in args) - - def test_warnings(self): - for toml_warnings in ['false', 'true', None]: - configure_args = [] - if toml_warnings is not None: - configure_args = ["--set", "rust.deny-warnings=" + toml_warnings] - - _, env = self.build_args(configure_args, args=["--warnings=warn"]) - self.assertFalse("-Dwarnings" in env["RUSTFLAGS"]) - - _, env = self.build_args(configure_args, args=["--warnings=deny"]) - self.assertTrue("-Dwarnings" in env["RUSTFLAGS"]) diff --git a/standalonex/src/build.rs b/standalonex/src/build.rs deleted file mode 100644 index e0e32d31..00000000 --- a/standalonex/src/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -use std::env; - -fn main() { - let host = env::var("HOST").unwrap(); - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rustc-env=BUILD_TRIPLE={host}"); -} diff --git a/standalonex/src/build_helper/src/stage0_parser.rs b/standalonex/src/build_helper/src/stage0_parser.rs index 2a0c12a1..b70462d6 100644 --- a/standalonex/src/build_helper/src/stage0_parser.rs +++ b/standalonex/src/build_helper/src/stage0_parser.rs @@ -1,4 +1,6 @@ use std::collections::BTreeMap; +use std::fs; +use std::path::Path; #[derive(Default, Clone)] pub struct Stage0 { @@ -24,8 +26,9 @@ pub struct Stage0Config { pub nightly_branch: String, } -pub fn parse_stage0_file() -> Stage0 { - let stage0_content = include_str!("../../stage0"); +pub fn parse_stage0_file(path: &Path) -> Stage0 { + let stage0_content = std::fs::read_to_string(path) + .expect(&format!("Failed to read stage0 file: {}", path.display())); let mut stage0 = Stage0::default(); for line in stage0_content.lines() { diff --git a/standalonex/src/configure.py b/standalonex/src/configure.py deleted file mode 100755 index 70f4e709..00000000 --- a/standalonex/src/configure.py +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env python - -# ignore-tidy-linelength - -from __future__ import absolute_import, division, print_function -import shlex -import sys -import os -rust_dir = os.path.dirname(os.path.abspath(__file__)) -rust_dir = os.path.dirname(rust_dir) -rust_dir = os.path.dirname(rust_dir) -sys.path.append(os.path.join(rust_dir, "src", "bootstrap")) -import bootstrap # noqa: E402 - - -class Option(object): - def __init__(self, name, rustbuild, desc, value): - self.name = name - self.rustbuild = rustbuild - self.desc = desc - self.value = value - - -options = [] - - -def o(*args): - options.append(Option(*args, value=False)) - - -def v(*args): - options.append(Option(*args, value=True)) - - -o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code") -o("docs", "build.docs", "build standard library documentation") -o("compiler-docs", "build.compiler-docs", "build compiler documentation") -o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") -o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests") -o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") -o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds") -o("local-rust", None, "use an installed rustc rather than downloading a snapshot") -v("local-rust-root", None, "set prefix for local rust binary") -o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version") -o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM") -o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") -o("rpath", "rust.rpath", "build rpaths into rustc itself") -o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests") -o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") -o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date") -o("vendor", "build.vendor", "enable usage of vendored Rust crates") -o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, dfsan, lsan, msan, tsan, hwasan)") -o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball") -o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo") -o("profiler", "build.profiler", "build the profiler runtime") -o("full-tools", None, "enable all tools") -o("lld", "rust.lld", "build lld") -o("llvm-bitcode-linker", "rust.llvm-bitcode-linker", "build llvm bitcode linker") -o("clang", "llvm.clang", "build clang") -o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") -o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") -o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") -o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") - -v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") -v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") -v("llvm-ldflags", "llvm.ldflags", "build LLVM with these extra linker flags") - -v("llvm-libunwind", "rust.llvm-libunwind", "use LLVM libunwind") - -# Optimization and debugging options. These may be overridden by the release -# channel, etc. -o("optimize-llvm", "llvm.optimize", "build optimized LLVM") -o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") -o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") -o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") -o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") -o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") -o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions") -o("overflow-checks", "rust.overflow-checks", "build with overflow checks") -o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks") -o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata") -v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code") -v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler") -v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library") -v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools") -v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest") -v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file") - -v("prefix", "install.prefix", "set installation prefix") -v("localstatedir", "install.localstatedir", "local state directory") -v("datadir", "install.datadir", "install data") -v("sysconfdir", "install.sysconfdir", "install system configuration files") -v("infodir", "install.infodir", "install additional info") -v("libdir", "install.libdir", "install libraries") -v("mandir", "install.mandir", "install man pages in PATH") -v("docdir", "install.docdir", "install documentation in PATH") -v("bindir", "install.bindir", "install binaries") - -v("llvm-root", None, "set LLVM root") -v("llvm-config", None, "set path to llvm-config") -v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") -v("python", "build.python", "set path to python") -v("android-ndk", "build.android-ndk", "set path to Android NDK") -v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", - "MUSL root installation directory (deprecated)") -v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", - "x86_64-unknown-linux-musl install directory") -v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root", - "i586-unknown-linux-musl install directory") -v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root", - "i686-unknown-linux-musl install directory") -v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root", - "arm-unknown-linux-musleabi install directory") -v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root", - "arm-unknown-linux-musleabihf install directory") -v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root", - "armv5te-unknown-linux-musleabi install directory") -v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root", - "armv7-unknown-linux-musleabi install directory") -v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root", - "armv7-unknown-linux-musleabihf install directory") -v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root", - "aarch64-unknown-linux-musl install directory") -v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root", - "mips-unknown-linux-musl install directory") -v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root", - "mipsel-unknown-linux-musl install directory") -v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root", - "mips64-unknown-linux-muslabi64 install directory") -v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root", - "mips64el-unknown-linux-muslabi64 install directory") -v("musl-root-riscv32gc", "target.riscv32gc-unknown-linux-musl.musl-root", - "riscv32gc-unknown-linux-musl install directory") -v("musl-root-riscv64gc", "target.riscv64gc-unknown-linux-musl.musl-root", - "riscv64gc-unknown-linux-musl install directory") -v("musl-root-loongarch64", "target.loongarch64-unknown-linux-musl.musl-root", - "loongarch64-unknown-linux-musl install directory") -v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs", - "rootfs in qemu testing, you probably don't want to use this") -v("experimental-targets", "llvm.experimental-targets", - "experimental LLVM targets to build") -v("release-channel", "rust.channel", "the name of the release channel to build") -v("release-description", "rust.description", "optional descriptive string for version output") -v("dist-compression-formats", None, "List of compression formats to use") - -# Used on systems where "cc" is unavailable -v("default-linker", "rust.default-linker", "the default linker") - -# Many of these are saved below during the "writing configuration" step -# (others are conditionally saved). -o("manage-submodules", "build.submodules", "let the build manage the git submodules") -o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)") -o("extended", "build.extended", "build an extended rust tool set") - -v("bootstrap-cache-path", None, "use provided path for the bootstrap cache") -v("tools", None, "List of extended tools will be installed") -v("codegen-backends", None, "List of codegen backends to build") -v("build", "build.build", "GNUs ./configure syntax LLVM build triple") -v("host", None, "List of GNUs ./configure syntax LLVM host triples") -v("target", None, "List of GNUs ./configure syntax LLVM target triples") - -# Options specific to this configure script -o("option-checking", None, "complain about unrecognized options in this configure script") -o("verbose-configure", None, "don't truncate options when printing them in this configure script") -v("set", None, "set arbitrary key/value pairs in TOML configuration") - - -def p(msg): - print("configure: " + msg) - - -def err(msg): - print("\nconfigure: ERROR: " + msg + "\n") - sys.exit(1) - -def is_value_list(key): - for option in options: - if option.name == key and option.desc.startswith('List of'): - return True - return False - -if '--help' in sys.argv or '-h' in sys.argv: - print('Usage: ./configure [options]') - print('') - print('Options') - for option in options: - if 'android' in option.name: - # no one needs to know about these obscure options - continue - if option.value: - print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc)) - else: - print('\t--enable-{:25} OR --disable-{}'.format(option.name, option.name)) - print('\t\t' + option.desc) - print('') - print('This configure script is a thin configuration shim over the true') - print('configuration system, `config.toml`. You can explore the comments') - print('in `config.example.toml` next to this configure script to see') - print('more information about what each option is. Additionally you can') - print('pass `--set` as an argument to set arbitrary key/value pairs') - print('in the TOML configuration if desired') - print('') - print('Also note that all options which take `--enable` can similarly') - print('be passed with `--disable-foo` to forcibly disable the option') - sys.exit(0) - -VERBOSE = False - -# Parse all command line arguments into one of these three lists, handling -# boolean and value-based options separately -def parse_args(args): - unknown_args = [] - need_value_args = [] - known_args = {} - - i = 0 - while i < len(args): - arg = args[i] - i += 1 - if not arg.startswith('--'): - unknown_args.append(arg) - continue - - found = False - for option in options: - value = None - if option.value: - keyval = arg[2:].split('=', 1) - key = keyval[0] - if option.name != key: - continue - - if len(keyval) > 1: - value = keyval[1] - elif i < len(args): - value = args[i] - i += 1 - else: - need_value_args.append(arg) - continue - else: - if arg[2:] == 'enable-' + option.name: - value = True - elif arg[2:] == 'disable-' + option.name: - value = False - else: - continue - - found = True - if option.name not in known_args: - known_args[option.name] = [] - known_args[option.name].append((option, value)) - break - - if not found: - unknown_args.append(arg) - - # NOTE: here and a few other places, we use [-1] to apply the *last* value - # passed. But if option-checking is enabled, then the known_args loop will - # also assert that options are only passed once. - option_checking = ('option-checking' not in known_args - or known_args['option-checking'][-1][1]) - if option_checking: - if len(unknown_args) > 0: - err("Option '" + unknown_args[0] + "' is not recognized") - if len(need_value_args) > 0: - err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) - - global VERBOSE - VERBOSE = 'verbose-configure' in known_args - - config = {} - - set('build.configure-args', args, config) - apply_args(known_args, option_checking, config) - return parse_example_config(known_args, config) - - -def build(known_args): - if 'build' in known_args: - return known_args['build'][-1][1] - return bootstrap.default_build_triple(verbose=False) - - -def set(key, value, config): - if isinstance(value, list): - # Remove empty values, which value.split(',') tends to generate and - # replace single quotes for double quotes to ensure correct parsing. - value = [v.replace('\'', '"') for v in value if v] - - s = "{:20} := {}".format(key, value) - if len(s) < 70 or VERBOSE: - p(s) - else: - p(s[:70] + " ...") - - arr = config - - # Split `key` on periods using shell semantics. - lexer = shlex.shlex(key, posix=True) - lexer.whitespace = "." - lexer.wordchars += "-" - parts = list(lexer) - - for i, part in enumerate(parts): - if i == len(parts) - 1: - if is_value_list(part) and isinstance(value, str): - value = value.split(',') - arr[part] = value - else: - if part not in arr: - arr[part] = {} - arr = arr[part] - - -def apply_args(known_args, option_checking, config): - for key in known_args: - # The `set` option is special and can be passed a bunch of times - if key == 'set': - for _option, value in known_args[key]: - keyval = value.split('=', 1) - if len(keyval) == 1 or keyval[1] == "true": - value = True - elif keyval[1] == "false": - value = False - else: - value = keyval[1] - set(keyval[0], value, config) - continue - - # Ensure each option is only passed once - arr = known_args[key] - if option_checking and len(arr) > 1: - err("Option '{}' provided more than once".format(key)) - option, value = arr[-1] - - # If we have a clear avenue to set our value in rustbuild, do so - if option.rustbuild is not None: - set(option.rustbuild, value, config) - continue - - # Otherwise we're a "special" option and need some extra handling, so do - # that here. - build_triple = build(known_args) - - if option.name == 'sccache': - set('llvm.ccache', 'sccache', config) - elif option.name == 'local-rust': - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/rustc'): - set('build.rustc', path + '/rustc', config) - break - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/cargo'): - set('build.cargo', path + '/cargo', config) - break - elif option.name == 'local-rust-root': - set('build.rustc', value + '/bin/rustc', config) - set('build.cargo', value + '/bin/cargo', config) - elif option.name == 'llvm-root': - set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config) - elif option.name == 'llvm-config': - set('target.{}.llvm-config'.format(build_triple), value, config) - elif option.name == 'llvm-filecheck': - set('target.{}.llvm-filecheck'.format(build_triple), value, config) - elif option.name == 'tools': - set('build.tools', value.split(','), config) - elif option.name == 'bootstrap-cache-path': - set('build.bootstrap-cache-path', value, config) - elif option.name == 'codegen-backends': - set('rust.codegen-backends', value.split(','), config) - elif option.name == 'host': - set('build.host', value.split(','), config) - elif option.name == 'target': - set('build.target', value.split(','), config) - elif option.name == 'full-tools': - set('rust.codegen-backends', ['llvm'], config) - set('rust.lld', True, config) - set('rust.llvm-tools', True, config) - set('rust.llvm-bitcode-linker', True, config) - set('build.extended', True, config) - elif option.name in ['option-checking', 'verbose-configure']: - # this was handled above - pass - elif option.name == 'dist-compression-formats': - set('dist.compression-formats', value.split(','), config) - else: - raise RuntimeError("unhandled option {}".format(option.name)) - -# "Parse" the `config.example.toml` file into the various sections, and we'll -# use this as a template of a `config.toml` to write out which preserves -# all the various comments and whatnot. -# -# Note that the `target` section is handled separately as we'll duplicate it -# per configured target, so there's a bit of special handling for that here. -def parse_example_config(known_args, config): - sections = {} - cur_section = None - sections[None] = [] - section_order = [None] - targets = {} - top_level_keys = [] - - with open(rust_dir + '/config.example.toml') as example_config: - example_lines = example_config.read().split("\n") - for line in example_lines: - if cur_section is None: - if line.count('=') == 1: - top_level_key = line.split('=')[0] - top_level_key = top_level_key.strip(' #') - top_level_keys.append(top_level_key) - if line.startswith('['): - cur_section = line[1:-1] - if cur_section.startswith('target'): - cur_section = 'target' - elif '.' in cur_section: - raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) - sections[cur_section] = [line] - section_order.append(cur_section) - else: - sections[cur_section].append(line) - - # Fill out the `targets` array by giving all configured targets a copy of the - # `target` section we just loaded from the example config - configured_targets = [build(known_args)] - if 'build' in config: - if 'host' in config['build']: - configured_targets += config['build']['host'] - if 'target' in config['build']: - configured_targets += config['build']['target'] - if 'target' in config: - for target in config['target']: - configured_targets.append(target) - for target in configured_targets: - targets[target] = sections['target'][:] - # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. - # Avoid using quotes unless it's necessary. - targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) - - if 'profile' not in config: - set('profile', 'dist', config) - configure_file(sections, top_level_keys, targets, config) - return section_order, sections, targets - - -def is_number(value): - try: - float(value) - return True - except ValueError: - return False - - -# Here we walk through the constructed configuration we have from the parsed -# command line arguments. We then apply each piece of configuration by -# basically just doing a `sed` to change the various configuration line to what -# we've got configure. -def to_toml(value): - if isinstance(value, bool): - if value: - return "true" - else: - return "false" - elif isinstance(value, list): - return '[' + ', '.join(map(to_toml, value)) + ']' - elif isinstance(value, str): - # Don't put quotes around numeric values - if is_number(value): - return value - else: - return "'" + value + "'" - elif isinstance(value, dict): - return "{" + ", ".join(map(lambda a: "{} = {}".format(to_toml(a[0]), to_toml(a[1])), value.items())) + "}" - else: - raise RuntimeError('no toml') - - -def configure_section(lines, config): - for key in config: - value = config[key] - found = False - for i, line in enumerate(lines): - if not line.startswith('#' + key + ' = '): - continue - found = True - lines[i] = "{} = {}".format(key, to_toml(value)) - break - if not found: - # These are used by rpm, but aren't accepted by x.py. - # Give a warning that they're ignored, but not a hard error. - if key in ["infodir", "localstatedir"]: - print("WARNING: {} will be ignored".format(key)) - else: - raise RuntimeError("failed to find config line for {}".format(key)) - - -def configure_top_level_key(lines, top_level_key, value): - for i, line in enumerate(lines): - if line.startswith('#' + top_level_key + ' = ') or line.startswith(top_level_key + ' = '): - lines[i] = "{} = {}".format(top_level_key, to_toml(value)) - return - - raise RuntimeError("failed to find config line for {}".format(top_level_key)) - - -# Modify `sections` to reflect the parsed arguments and example configs. -def configure_file(sections, top_level_keys, targets, config): - for section_key, section_config in config.items(): - if section_key not in sections and section_key not in top_level_keys: - raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) - if section_key in top_level_keys: - configure_top_level_key(sections[None], section_key, section_config) - - elif section_key == 'target': - for target in section_config: - configure_section(targets[target], section_config[target]) - else: - configure_section(sections[section_key], section_config) - - -def write_uncommented(target, f): - block = [] - is_comment = True - - for line in target: - block.append(line) - if len(line) == 0: - if not is_comment: - for ln in block: - f.write(ln + "\n") - block = [] - is_comment = True - continue - is_comment = is_comment and line.startswith('#') - return f - - -def write_config_toml(writer, section_order, targets, sections): - for section in section_order: - if section == 'target': - for target in targets: - writer = write_uncommented(targets[target], writer) - else: - writer = write_uncommented(sections[section], writer) - -def quit_if_file_exists(file): - if os.path.isfile(file): - msg = "Existing '{}' detected. Exiting".format(file) - - # If the output object directory isn't empty, we can get these errors - host_objdir = os.environ.get("OBJDIR_ON_HOST") - if host_objdir is not None: - msg += "\nIs objdir '{}' clean?".format(host_objdir) - - err(msg) - -if __name__ == "__main__": - # If 'config.toml' already exists, exit the script at this point - quit_if_file_exists('config.toml') - - if "GITHUB_ACTIONS" in os.environ: - print("::group::Configure the build") - p("processing command line") - # Parse all known arguments into a configuration structure that reflects the - # TOML we're going to write out - p("") - section_order, sections, targets = parse_args(sys.argv[1:]) - - # Now that we've built up our `config.toml`, write it all out in the same - # order that we read it in. - p("") - p("writing `config.toml` in current directory") - with bootstrap.output('config.toml') as f: - write_config_toml(f, section_order, targets, sections) - - with bootstrap.output('Makefile') as f: - contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') - contents = open(contents).read() - contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') - contents = contents.replace("$(CFG_PYTHON)", sys.executable) - f.write(contents) - - p("") - p("run `python {}/x.py --help`".format(rust_dir)) - if "GITHUB_ACTIONS" in os.environ: - print("::endgroup::") diff --git a/standalonex/src/defaults/README.md b/standalonex/src/defaults/README.md deleted file mode 100644 index f5b96db1..00000000 --- a/standalonex/src/defaults/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# About bootstrap defaults - -These defaults are intended to be a good starting point for working with x.py, -with the understanding that no one set of defaults make sense for everyone. - -They are still experimental, and we'd appreciate your help improving them! -If you use a setting that's not in these defaults that you think -others would benefit from, please [file an issue] or make a PR with the changes. -Similarly, if one of these defaults doesn't match what you use personally, -please open an issue to get it changed. - -[file an issue]: https://github.com/rust-lang/rust/issues/new/choose diff --git a/standalonex/src/flake.nix b/standalonex/src/flake.nix new file mode 100644 index 00000000..48a40197 --- /dev/null +++ b/standalonex/src/flake.nix @@ -0,0 +1,72 @@ +{ + description = "Nix flake for the Rust bootstrap workspace"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + flake-utils.url = "github:meta-introspector/flake-utils?ref=feature/CRQ-016-nixify"; + cargo2nix.url = "github:meta-introspector/cargo2nix?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, flake-utils, cargo2nix, rust-overlay }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ + rust-overlay.overlays.default + cargo2nix.overlays.default # Apply cargo2nix overlay here + ]; + }; + + rustVersion = "1.84.1"; # Explicitly set rust version + rustPkgs = pkgs.rustBuilder.makePackageSet { + inherit rustVersion; + packageFun = args: (import ./Cargo.nix) (args // { + lib = pkgs.lib; + hostPlatform = pkgs.stdenv.hostPlatform; + rustLib = pkgs.rustPlatform; + mkRustCrate = pkgs.rustPlatform.buildRustPackage; + workspaceSrc = ./.; + ignoreLockHash = false; + cargo2nix = cargo2nix; # Pass cargo2nix itself + }); + packageOverrides = pkgs.rustBuilder.overrides.make (final: prev: { + globset = prev.globset.overrideAttrs (old: { + version = "0.4.16"; + src = pkgs.rustBuilder.fetchCratesIo { + name = "globset"; + version = "0.4.16"; + sha256 = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"; # SHA256 for globset 0.4.16 + }; + }); + }); + }; + + bootstrapApp = rustPkgs.workspace.bootstrap; + buildHelperApp = rustPkgs.workspace.build_helper; + in + { + packages = { + bootstrap = bootstrapApp { }; # Call the function to get the derivation + build_helper = buildHelperApp { }; # Call the function to get the derivation + default = bootstrapApp { }; # Call the function to get the derivation + }; + + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.rust-bin.stable.${rustVersion}.default + pkgs.cargo + pkgs.rustc + pkgs.rustfmt + pkgs.clippy + pkgs.git + pkgs.pkg-config + pkgs.cmake + pkgs.libiconv # For macOS + ]; + CARGO_HOME = "${pkgs.writeText "cargo-home" ""}"; # Prevent cargo from writing to ~/.cargo + RUST_SRC_PATH = "${pkgs.rustPlatform.rustLibSrc}"; + }; + }); +} diff --git a/standalonex/src/src/bin/main.rs b/standalonex/src/src/bin/main.rs deleted file mode 100644 index 409a644b..00000000 --- a/standalonex/src/src/bin/main.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! bootstrap, the Rust build system -//! -//! This is the entry point for the build system used to compile the `rustc` -//! compiler. Lots of documentation can be found in the `README.md` file in the -//! parent directory, and otherwise documentation can be found throughout the `build` -//! directory in each respective module. - -use std::fs::{self, OpenOptions}; -use std::io::{self, BufRead, BufReader, IsTerminal, Write}; -use std::str::FromStr; -use std::{env, process}; - -use bootstrap::{ - Build, CONFIG_CHANGE_HISTORY, Config, Flags, Subcommand, find_recent_config_change_ids, - human_readable_changes, t, -}; -use build_helper::ci::CiEnv; - -fn main() { - let args = env::args().skip(1).collect::>(); - - if Flags::try_parse_verbose_help(&args) { - return; - } - - let flags = Flags::parse(&args); - let config = Config::parse(flags); - - let mut build_lock; - let _build_lock_guard; - - if !config.bypass_bootstrap_lock { - // Display PID of process holding the lock - // PID will be stored in a lock file - let lock_path = config.out.join("lock"); - let pid = fs::read_to_string(&lock_path); - - build_lock = fd_lock::RwLock::new(t!(fs::OpenOptions::new() - .write(true) - .truncate(true) - .create(true) - .open(&lock_path))); - _build_lock_guard = match build_lock.try_write() { - Ok(mut lock) => { - t!(lock.write(process::id().to_string().as_ref())); - lock - } - err => { - drop(err); - if let Ok(pid) = pid { - println!("WARNING: build directory locked by process {pid}, waiting for lock"); - } else { - println!("WARNING: build directory locked, waiting for lock"); - } - let mut lock = t!(build_lock.write()); - t!(lock.write(process::id().to_string().as_ref())); - lock - } - }; - } - - // check_version warnings are not printed during setup, or during CI - let changelog_suggestion = if matches!(config.cmd, Subcommand::Setup { .. }) || CiEnv::is_ci() { - None - } else { - check_version(&config) - }; - - // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the - // changelog warning, not the `x.py setup` message. - let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); - if suggest_setup { - println!("WARNING: you have not made a `config.toml`"); - println!( - "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); - let dump_bootstrap_shims = config.dump_bootstrap_shims; - let out_dir = config.out.clone(); - - Build::new(config).build(); - - if suggest_setup { - println!("WARNING: you have not made a `config.toml`"); - println!( - "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - // Give a warning if the pre-commit script is in pre-commit and not pre-push. - // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. - // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. - // Instead, look for this comment, which is almost certainly not in any custom hook. - if fs::read_to_string(pre_commit).map_or(false, |contents| { - contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") - }) { - println!( - "WARNING: You have the pre-push script installed to .git/hooks/pre-commit. \ - Consider moving it to .git/hooks/pre-push instead, which runs less often." - ); - } - - if suggest_setup || changelog_suggestion.is_some() { - println!("NOTE: this message was printed twice to make it more likely to be seen"); - } - - if dump_bootstrap_shims { - let dump_dir = out_dir.join("bootstrap-shims-dump"); - assert!(dump_dir.exists()); - - for entry in walkdir::WalkDir::new(&dump_dir) { - let entry = t!(entry); - - if !entry.file_type().is_file() { - continue; - } - - let file = t!(fs::File::open(entry.path())); - - // To ensure deterministic results we must sort the dump lines. - // This is necessary because the order of rustc invocations different - // almost all the time. - let mut lines: Vec = t!(BufReader::new(&file).lines().collect()); - lines.sort_by_key(|t| t.to_lowercase()); - let mut file = t!(OpenOptions::new().write(true).truncate(true).open(entry.path())); - t!(file.write_all(lines.join("\n").as_bytes())); - } - } -} - -fn check_version(config: &Config) -> Option { - let mut msg = String::new(); - - let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap().change_id; - let warned_id_path = config.out.join("bootstrap").join(".last-warned-change-id"); - - if let Some(mut id) = config.change_id { - if id == latest_change_id { - return None; - } - - // Always try to use `change-id` from .last-warned-change-id first. If it doesn't exist, - // then use the one from the config.toml. This way we never show the same warnings - // more than once. - if let Ok(t) = fs::read_to_string(&warned_id_path) { - let last_warned_id = usize::from_str(&t) - .unwrap_or_else(|_| panic!("{} is corrupted.", warned_id_path.display())); - - // We only use the last_warned_id if it exists in `CONFIG_CHANGE_HISTORY`. - // Otherwise, we may retrieve all the changes if it's not the highest value. - // For better understanding, refer to `change_tracker::find_recent_config_change_ids`. - if CONFIG_CHANGE_HISTORY.iter().any(|config| config.change_id == last_warned_id) { - id = last_warned_id; - } - }; - - let changes = find_recent_config_change_ids(id); - - if changes.is_empty() { - return None; - } - - msg.push_str("There have been changes to x.py since you last updated:\n"); - msg.push_str(&human_readable_changes(&changes)); - - msg.push_str("NOTE: to silence this warning, "); - msg.push_str(&format!( - "update `config.toml` to use `change-id = {latest_change_id}` instead" - )); - - if io::stdout().is_terminal() && !config.dry_run() { - t!(fs::write(warned_id_path, latest_change_id.to_string())); - } - } else { - msg.push_str("WARNING: The `change-id` is missing in the `config.toml`. This means that you will not be able to track the major changes made to the bootstrap configurations.\n"); - msg.push_str("NOTE: to silence this warning, "); - msg.push_str(&format!("add `change-id = {latest_change_id}` at the top of `config.toml`")); - }; - - Some(msg) -} diff --git a/standalonex/src/src/bin/rustc.rs b/standalonex/src/src/bin/rustc.rs deleted file mode 100644 index 88595ff7..00000000 --- a/standalonex/src/src/bin/rustc.rs +++ /dev/null @@ -1,454 +0,0 @@ -//! Shim which is passed to Cargo as "rustc" when running the bootstrap. -//! -//! This shim will take care of some various tasks that our build process -//! requires that Cargo can't quite do through normal configuration: -//! -//! 1. When compiling build scripts and build dependencies, we need a guaranteed -//! full standard library available. The only compiler which actually has -//! this is the snapshot, so we detect this situation and always compile with -//! the snapshot compiler. -//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling -//! (and this slightly differs based on a whether we're using a snapshot or -//! not), so we do that all here. -//! -//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of -//! switching compilers for the bootstrap and for build scripts will probably -//! never get replaced. - -use std::env; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command}; -use std::time::Instant; - -use shared_helpers::{ - dylib_path, dylib_path_var, exe, maybe_dump, parse_rustc_stage, parse_rustc_verbose, - parse_value_from_args, -}; - -#[path = "../utils/shared_helpers.rs"] -mod shared_helpers; - -fn main() { - let orig_args = env::args_os().skip(1).collect::>(); - let mut args = orig_args.clone(); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = parse_value_from_args(&orig_args, "--target"); - let version = args.iter().find(|w| &**w == "-vV"); - - // Use a different compiler for build scripts, since there may not yet be a - // libstd for the real compiler to use. However, if Cargo is attempting to - // determine the version of the compiler, the real compiler needs to be - // used. Currently, these two states are differentiated based on whether - // --target and -vV is/isn't passed. - let is_build_script = target.is_none() && version.is_none(); - let (rustc, libdir) = if is_build_script { - ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") - } else { - ("RUSTC_REAL", "RUSTC_LIBDIR") - }; - - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); - - let rustc_real = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); - let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&libdir)); - - // if we're running clippy, trust cargo-clippy to set clippy-driver appropriately (and don't override it with rustc). - // otherwise, substitute whatever cargo thinks rustc should be with RUSTC_REAL. - // NOTE: this means we ignore RUSTC in the environment. - // FIXME: We might want to consider removing RUSTC_REAL and setting RUSTC directly? - // NOTE: we intentionally pass the name of the host, not the target. - let host = env::var("CFG_COMPILER_BUILD_TRIPLE").unwrap(); - let is_clippy = args[0].to_string_lossy().ends_with(&exe("clippy-driver", &host)); - let rustc_driver = if is_clippy { - if is_build_script { - // Don't run clippy on build scripts (for one thing, we may not have libstd built with - // the appropriate version yet, e.g. for stage 1 std). - // Also remove the `clippy-driver` param in addition to the RUSTC param. - args.drain(..2); - rustc_real - } else { - args.remove(0) - } - } else { - // Cargo doesn't respect RUSTC_WRAPPER for version information >:( - // don't remove the first arg if we're being run as RUSTC instead of RUSTC_WRAPPER. - // Cargo also sometimes doesn't pass the `.exe` suffix on Windows - add it manually. - let current_exe = env::current_exe().expect("couldn't get path to rustc shim"); - let arg0 = exe(args[0].to_str().expect("only utf8 paths are supported"), &host); - if Path::new(&arg0) == current_exe { - args.remove(0); - } - rustc_real - }; - - // Get the name of the crate we're compiling, if any. - let crate_name = parse_value_from_args(&orig_args, "--crate-name"); - - // When statically linking `std` into `rustc_driver`, remove `-C prefer-dynamic` - if env::var("RUSTC_LINK_STD_INTO_RUSTC_DRIVER").unwrap() == "1" - && crate_name == Some("rustc_driver") - { - if let Some(pos) = args.iter().enumerate().position(|(i, a)| { - a == "-C" && args.get(i + 1).map(|a| a == "prefer-dynamic").unwrap_or(false) - }) { - args.remove(pos); - args.remove(pos); - } - if let Some(pos) = args.iter().position(|a| a == "-Cprefer-dynamic") { - args.remove(pos); - } - } - - let mut cmd = match env::var_os("RUSTC_WRAPPER_REAL") { - Some(wrapper) if !wrapper.is_empty() => { - let mut cmd = Command::new(wrapper); - cmd.arg(rustc_driver); - cmd - } - _ => Command::new(rustc_driver), - }; - cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - if let Some(crate_name) = crate_name { - if let Some(target) = env::var_os("RUSTC_TIME") { - if target == "all" - || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) - { - cmd.arg("-Ztime-passes"); - } - } - } - - // Print backtrace in case of ICE - if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { - cmd.env("RUST_BACKTRACE", "1"); - } - - if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { - cmd.args(lint_flags.split_whitespace()); - } - - // Conditionally pass `-Zon-broken-pipe=kill` to underlying rustc. Not all binaries want - // `-Zon-broken-pipe=kill`, which includes cargo itself. - if env::var_os("FORCE_ON_BROKEN_PIPE_KILL").is_some() { - cmd.arg("-Z").arg("on-broken-pipe=kill"); - } - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - - // If we're compiling specifically the `panic_abort` crate then we pass - // the `-C panic=abort` option. Note that we do not do this for any - // other crate intentionally as this is the only crate for now that we - // ship with panic=abort. - // - // This... is a bit of a hack how we detect this. Ideally this - // information should be encoded in the crate I guess? Would likely - // require an RFC amendment to RFC 1513, however. - if crate_name == Some("panic_abort") { - cmd.arg("-C").arg("panic=abort"); - } - - let crate_type = parse_value_from_args(&orig_args, "--crate-type"); - // `-Ztls-model=initial-exec` must not be applied to proc-macros, see - // issue https://github.com/rust-lang/rust/issues/100530 - if env::var("RUSTC_TLS_MODEL_INITIAL_EXEC").is_ok() - && crate_type != Some("proc-macro") - && !matches!(crate_name, Some("proc_macro2" | "quote" | "syn" | "synstructure")) - { - cmd.arg("-Ztls-model=initial-exec"); - } - } else { - // Find any host flags that were passed by bootstrap. - // The flags are stored in a RUSTC_HOST_FLAGS variable, separated by spaces. - if let Ok(flags) = std::env::var("RUSTC_HOST_FLAGS") { - cmd.args(flags.split(' ')); - } - } - - if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { - cmd.arg("--remap-path-prefix").arg(&map); - } - // The remap flags for Cargo registry sources need to be passed after the remapping for the - // Rust source code directory, to handle cases when $CARGO_HOME is inside the source directory. - if let Ok(maps) = env::var("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP") { - for map in maps.split('\t') { - cmd.arg("--remap-path-prefix").arg(map); - } - } - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. We also do this for host crates, since those - // may be proc macros, in which case we might ship them. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - - // allow-features is handled from within this rustc wrapper because of - // issues with build scripts. Some packages use build scripts to - // dynamically detect if certain nightly features are available. - // There are different ways this causes problems: - // - // * rustix runs `rustc` on a small test program to see if the feature is - // available (and sets a `cfg` if it is). It does not honor - // CARGO_ENCODED_RUSTFLAGS. - // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables - // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for - // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set - // for build-dependencies when --target is used. - // - // The issues above means we can't just use RUSTFLAGS, and we can't use - // `cargo -Zallow-features=â€Ļ`. Passing it through here ensures that it - // always gets set. Unfortunately that also means we need to enable more - // features than we really want (like those for proc-macro2), but there - // isn't much of a way around it. - // - // I think it is unfortunate that build scripts are doing this at all, - // since changes to nightly features can cause crates to break even if the - // user didn't want or care about the use of the nightly features. I think - // nightly features should be opt-in only. Unfortunately the dynamic - // checks are now too wide spread that we just need to deal with it. - // - // If you want to try to remove this, I suggest working with the crate - // authors to remove the dynamic checking. Another option is to pursue - // https://github.com/rust-lang/cargo/issues/11244 and - // https://github.com/rust-lang/cargo/issues/4423, which will likely be - // very difficult, but could help expose -Zallow-features into build - // scripts so they could try to honor them. - if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { - cmd.arg(format!("-Zallow-features={allow_features}")); - } - - if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { - for flag in flags.split(' ') { - cmd.arg(flag); - } - } - - if env::var_os("RUSTC_BOLT_LINK_FLAGS").is_some() { - if let Some("rustc_driver") = crate_name { - cmd.arg("-Clink-args=-Wl,-q"); - } - } - - let is_test = args.iter().any(|a| a == "--test"); - if verbose > 2 { - let rust_env_vars = - env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); - let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; - let prefix = match crate_name { - Some(crate_name) => format!("{prefix} {crate_name}"), - None => prefix.to_string(), - }; - for (i, (k, v)) in rust_env_vars.enumerate() { - eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); - } - eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); - eprintln!( - "{} command: {:?}={:?} {:?}", - prefix, - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("{prefix} sysroot: {sysroot:?}"); - eprintln!("{prefix} libdir: {libdir:?}"); - } - - maybe_dump(format!("stage{stage}-rustc"), &cmd); - - let start = Instant::now(); - let (child, status) = { - let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); - let mut child = cmd.spawn().expect(&errmsg); - let status = child.wait().expect(&errmsg); - (child, status) - }; - - if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() - || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() - { - if let Some(crate_name) = crate_name { - let dur = start.elapsed(); - // If the user requested resource usage data, then - // include that in addition to the timing output. - let rusage_data = - env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); - eprintln!( - "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", - crate_name, - is_test, - dur.as_secs(), - dur.subsec_millis(), - if rusage_data.is_some() { " " } else { "" }, - rusage_data.unwrap_or_default(), - ); - } - } - - if status.success() { - std::process::exit(0); - // NOTE: everything below here is unreachable. do not put code that - // should run on success, after this block. - } - if verbose > 0 { - println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); - } - - if let Some(mut on_fail) = on_fail { - on_fail.status().expect("Could not run the on_fail command"); - } - - // Preserve the exit code. In case of signal, exit with 0xfe since it's - // awkward to preserve this status in a cross-platform way. - match status.code() { - Some(i) => std::process::exit(i), - None => { - eprintln!("rustc exited with {status}"); - std::process::exit(0xfe); - } - } -} - -#[cfg(all(not(unix), not(windows)))] -// In the future we can add this for more platforms -fn format_rusage_data(_child: Child) -> Option { - None -} - -#[cfg(windows)] -fn format_rusage_data(child: Child) -> Option { - use std::os::windows::io::AsRawHandle; - - use windows::Win32::Foundation::HANDLE; - use windows::Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}; - use windows::Win32::System::Threading::GetProcessTimes; - use windows::Win32::System::Time::FileTimeToSystemTime; - - let handle = HANDLE(child.as_raw_handle() as isize); - - let mut user_filetime = Default::default(); - let mut user_time = Default::default(); - let mut kernel_filetime = Default::default(); - let mut kernel_time = Default::default(); - let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); - let memory_counters_size = std::mem::size_of_val(&memory_counters); - - unsafe { - GetProcessTimes( - handle, - &mut Default::default(), - &mut Default::default(), - &mut kernel_filetime, - &mut user_filetime, - ) - } - .ok()?; - unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok()?; - unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok()?; - - // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process - // with the given handle and none of that process's children. - unsafe { K32GetProcessMemoryInfo(handle, &mut memory_counters, memory_counters_size as u32) } - .ok() - .ok()?; - - // Guide on interpreting these numbers: - // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information - let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; - let peak_page_file = memory_counters.PeakPagefileUsage / 1024; - let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; - let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; - Some(format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - peak working set (kb): {PEAK_WORKING_SET} \ - peak page file usage (kb): {PEAK_PAGE_FILE} \ - peak paged pool usage (kb): {PEAK_PAGED_POOL} \ - peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ - page faults: {PAGE_FAULTS}", - USER_SEC = user_time.wSecond + (user_time.wMinute * 60), - USER_USEC = user_time.wMilliseconds, - SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), - SYS_USEC = kernel_time.wMilliseconds, - PEAK_WORKING_SET = peak_working_set, - PEAK_PAGE_FILE = peak_page_file, - PEAK_PAGED_POOL = peak_paged_pool, - PEAK_NONPAGED_POOL = peak_nonpaged_pool, - PAGE_FAULTS = memory_counters.PageFaultCount, - )) -} - -#[cfg(unix)] -/// Tries to build a string with human readable data for several of the rusage -/// fields. Note that we are focusing mainly on data that we believe to be -/// supplied on Linux (the `rusage` struct has other fields in it but they are -/// currently unsupported by Linux). -fn format_rusage_data(_child: Child) -> Option { - let rusage: libc::rusage = unsafe { - let mut recv = std::mem::zeroed(); - // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children - // (and grandchildren, etc) processes that have respectively terminated - // and been waited for. - let retval = libc::getrusage(-1, &mut recv); - if retval != 0 { - return None; - } - recv - }; - // Mac OS X reports the maxrss in bytes, not kb. - let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; - let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; - - let mut init_str = format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - max rss (kb): {MAXRSS}", - USER_SEC = rusage.ru_utime.tv_sec, - USER_USEC = rusage.ru_utime.tv_usec, - SYS_SEC = rusage.ru_stime.tv_sec, - SYS_USEC = rusage.ru_stime.tv_usec, - MAXRSS = maxrss - ); - - // The remaining rusage stats vary in platform support. So we treat - // uniformly zero values in each category as "not worth printing", since it - // either means no events of that type occurred, or that the platform - // does not support it. - - let minflt = rusage.ru_minflt; - let majflt = rusage.ru_majflt; - if minflt != 0 || majflt != 0 { - init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); - } - - let inblock = rusage.ru_inblock; - let oublock = rusage.ru_oublock; - if inblock != 0 || oublock != 0 { - init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); - } - - let nvcsw = rusage.ru_nvcsw; - let nivcsw = rusage.ru_nivcsw; - if nvcsw != 0 || nivcsw != 0 { - init_str.push_str(&format!( - " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" - )); - } - - Some(init_str) -} diff --git a/standalonex/src/src/bin/rustdoc.rs b/standalonex/src/src/bin/rustdoc.rs deleted file mode 100644 index a338b9c8..00000000 --- a/standalonex/src/src/bin/rustdoc.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. -//! -//! See comments in `src/bootstrap/rustc.rs` for more information. - -use std::env; -use std::path::PathBuf; -use std::process::Command; - -use shared_helpers::{ - dylib_path, dylib_path_var, maybe_dump, parse_rustc_stage, parse_rustc_verbose, - parse_value_from_args, -}; - -#[path = "../utils/shared_helpers.rs"] -mod shared_helpers; - -fn main() { - let args = env::args_os().skip(1).collect::>(); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); - let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = parse_value_from_args(&args, "--target"); - - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(libdir.clone())); - - let mut cmd = Command::new(rustdoc); - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - } - - cmd.args(&args); - cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - - maybe_dump(format!("stage{stage}-rustdoc"), &cmd); - - if verbose > 1 { - eprintln!( - "rustdoc command: {:?}={:?} {:?}", - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("sysroot: {sysroot:?}"); - eprintln!("libdir: {libdir:?}"); - } - - std::process::exit(match cmd.status() { - Ok(s) => s.code().unwrap_or(1), - Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), - }) -} diff --git a/standalonex/src/src/bin/sccache-plus-cl.rs b/standalonex/src/src/bin/sccache-plus-cl.rs deleted file mode 100644 index 6e87d422..00000000 --- a/standalonex/src/src/bin/sccache-plus-cl.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::env; -use std::process::{self, Command}; - -fn main() { - let target = env::var("SCCACHE_TARGET").unwrap(); - // Locate the actual compiler that we're invoking - env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); - env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .out_dir("/") - .target(&target) - .host(&target) - .opt_level(0) - .warnings(false) - .debug(false); - let compiler = cfg.get_compiler(); - - // Invoke sccache with said compiler - let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); - let mut cmd = Command::new(sccache_path); - cmd.arg(compiler.path()); - for (k, v) in compiler.env() { - cmd.env(k, v); - } - for arg in env::args().skip(1) { - cmd.arg(arg); - } - - if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { - for s in s.split_whitespace() { - cmd.arg(s); - } - } - - let status = cmd.status().expect("failed to spawn"); - process::exit(status.code().unwrap_or(2)) -} diff --git a/standalonex/src/src/core/build_steps/check.rs b/standalonex/src/src/core/build_steps/check.rs deleted file mode 100644 index d46c0ab7..00000000 --- a/standalonex/src/src/core/build_steps/check.rs +++ /dev/null @@ -1,536 +0,0 @@ -//! Implementation of compiling the compiler and standard library, in "check"-based modes. - -use std::path::PathBuf; - -use crate::core::build_steps::compile::{ - add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, -}; -use crate::core::build_steps::tool::{SourceType, prepare_tool_cargo}; -use crate::core::builder::{ - self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, crate_description, -}; -use crate::core::config::TargetSelection; -use crate::{Compiler, Mode, Subcommand}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc - crates: Vec, - /// Override `Builder::kind` on cargo invocations. - /// - /// By default, `Builder::kind` is propagated as the subcommand to the cargo invocations. - /// However, there are cases when this is not desirable. For example, when running `x clippy $tool_name`, - /// passing `Builder::kind` to cargo invocations would run clippy on the entire compiler and library, - /// which is not useful if we only want to lint a few crates with specific rules. - override_build_kind: Option, -} - -impl Std { - pub fn new_with_build_kind(target: TargetSelection, kind: Option) -> Self { - Self { target, crates: vec![], override_build_kind: kind } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = std_crates_for_run_make(&run); - run.builder.ensure(Std { target: run.target, crates, override_build_kind: None }); - } - - fn run(self, builder: &Builder<'_>) { - builder.require_submodule("library/stdarch", None); - - let target = self.target; - let compiler = builder.compiler(builder.top_stage, builder.config.build); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - self.override_build_kind.unwrap_or(builder.kind), - ); - - std_cargo(builder, target, compiler.stage, &mut cargo); - if matches!(builder.config.cmd, Subcommand::Fix { .. }) { - // By default, cargo tries to fix all targets. Tell it not to fix tests until we've added `test` to the sysroot. - cargo.arg("--lib"); - } - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("library artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &libstd_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - // We skip populating the sysroot in non-zero stage because that'll lead - // to rlib/rmeta conflicts if std gets built during this session. - if compiler.stage == 0 { - let libdir = builder.sysroot_target_libdir(compiler, target); - let hostdir = builder.sysroot_target_libdir(compiler, compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - } - drop(_guard); - - // don't run on std twice with x.py clippy - // don't check test dependencies if we haven't built libtest - if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { - return; - } - - // Then run cargo again, once we've put the rmeta files for the library - // crates into the sysroot. This is needed because e.g., core's tests - // depend on `libtest` -- Cargo presumes it will exist, but it doesn't - // since we initialize with an empty sysroot. - // - // Currently only the "libtest" tree of crates does this. - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - self.override_build_kind.unwrap_or(builder.kind), - ); - - // If we're not in stage 0, tests and examples will fail to compile - // from `core` definitions being loaded from two different `libcore` - // .rmeta and .rlib files. - if compiler.stage == 0 { - cargo.arg("--all-targets"); - } - - std_cargo(builder, target, compiler.stage, &mut cargo); - - // Explicitly pass -p for all dependencies krates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check("library test/bench/example targets", target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &libstd_test_stamp(builder, compiler, target), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc - crates: Vec, - /// Override `Builder::kind` on cargo invocations. - /// - /// By default, `Builder::kind` is propagated as the subcommand to the cargo invocations. - /// However, there are cases when this is not desirable. For example, when running `x clippy $tool_name`, - /// passing `Builder::kind` to cargo invocations would run clippy on the entire compiler and library, - /// which is not useful if we only want to lint a few crates with specific rules. - override_build_kind: Option, -} - -impl Rustc { - pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { - Self::new_with_build_kind(target, builder, None) - } - - pub fn new_with_build_kind( - target: TargetSelection, - builder: &Builder<'_>, - kind: Option, - ) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { target, crates, override_build_kind: kind } - } -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Compiler); - run.builder.ensure(Rustc { target: run.target, crates, override_build_kind: None }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - if compiler.stage != 0 { - // If we're not in stage 0, then we won't have a std from the beta - // compiler around. That means we need to make sure there's one in - // the sysroot for the compiler to find. Otherwise, we're going to - // fail when building crates that need to generate code (e.g., build - // scripts and their dependencies). - builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); - } else { - builder.ensure(Std::new_with_build_kind(target, self.override_build_kind)); - } - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Rustc, - SourceType::InTree, - target, - self.override_build_kind.unwrap_or(builder.kind), - ); - - rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - // Explicitly pass -p for all compiler crates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("compiler artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &librustc_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - let libdir = builder.sysroot_target_libdir(compiler, target); - let hostdir = builder.sysroot_target_libdir(compiler, compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub backend: &'static str, -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - for &backend in &["cranelift", "gcc"] { - run.builder.ensure(CodegenBackend { target: run.target, backend }); - } - } - - fn run(self, builder: &Builder<'_>) { - // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved - if builder.build.config.vendor && self.backend == "gcc" { - println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); - return; - } - - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Codegen, - SourceType::InTree, - target, - builder.kind, - ); - - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let _guard = builder.msg_check(backend, target); - - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &codegen_backend_stamp(builder, compiler, target, backend), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind, - "src/tools/rust-analyzer", - SourceType::InTree, - &["in-rust-tree".to_owned()], - ); - - cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); - - // For ./x.py clippy, don't check those targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - // can't use `--all-targets` because `--examples` doesn't work well - cargo.arg("--bins"); - cargo.arg("--tests"); - cargo.arg("--benches"); - } - - let _guard = builder.msg_check("rust-analyzer artifacts", target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::ToolRustc, target).join(".rust-analyzer-check.stamp") - } - } -} - -macro_rules! tool_check_step { - ( - $name:ident, - $display_name:literal, - $path:literal, - $($alias:literal, )* - $source_type:path - $(, $default:literal )? - ) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const ONLY_HOSTS: bool = true; - /// don't ever check out-of-tree tools by default, they'll fail when toolstate is broken - const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&[ $path, $($alias),* ]) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind, - $path, - $source_type, - &[], - ); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - let _guard = builder.msg_check(&format!("{} artifacts", $display_name), target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - ) -> PathBuf { - builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) - } - } - } - }; -} - -tool_check_step!(Rustdoc, "rustdoc", "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); -// Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead -// of a submodule. Since the SourceType only drives the deny-warnings -// behavior, treat it as in-tree so that any new warnings in clippy will be -// rejected. -tool_check_step!(Clippy, "clippy", "src/tools/clippy", SourceType::InTree); -tool_check_step!(Miri, "miri", "src/tools/miri", SourceType::InTree); -tool_check_step!(CargoMiri, "cargo-miri", "src/tools/miri/cargo-miri", SourceType::InTree); -tool_check_step!(Rls, "rls", "src/tools/rls", SourceType::InTree); -tool_check_step!(Rustfmt, "rustfmt", "src/tools/rustfmt", SourceType::InTree); -tool_check_step!( - MiroptTestTools, - "miropt-test-tools", - "src/tools/miropt-test-tools", - SourceType::InTree -); -tool_check_step!( - TestFloatParse, - "test-float-parse", - "src/etc/test-float-parse", - SourceType::InTree -); - -tool_check_step!(Bootstrap, "bootstrap", "src/bootstrap", SourceType::InTree, false); - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_test_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: &str, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}-check.stamp")) -} diff --git a/standalonex/src/src/core/build_steps/clean.rs b/standalonex/src/src/core/build_steps/clean.rs deleted file mode 100644 index d857de96..00000000 --- a/standalonex/src/src/core/build_steps/clean.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! `./x.py clean` -//! -//! Responsible for cleaning out a build directory of all old and stale -//! artifacts to prepare for a fresh build. Currently doesn't remove the -//! `build/cache` directory (download cache) or the `build/$target/llvm` -//! directory unless the `--all` flag is present. - -use std::fs; -use std::io::{self, ErrorKind}; -use std::path::Path; - -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step, crate_description}; -use crate::utils::helpers::t; -use crate::{Build, Compiler, Kind, Mode, Subcommand}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CleanAll {} - -impl Step for CleanAll { - const DEFAULT: bool = true; - type Output = (); - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CleanAll {}) - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Subcommand::Clean { all, stage } = builder.config.cmd else { - unreachable!("wrong subcommand?") - }; - - if all && stage.is_some() { - panic!("--all and --stage can't be used at the same time for `x clean`"); - } - - clean(builder.build, all, stage) - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() // handled by DEFAULT - } -} - -macro_rules! clean_crate_tree { - ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - crates: Vec, - } - - impl Step for $name { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let crates = run.builder.in_tree_crates($root_crate, None); - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.target); - builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler = self.compiler; - let target = compiler.host; - let mut cargo = builder.bare_cargo(compiler, $mode, target, Kind::Clean); - - // Since https://github.com/rust-lang/rust/pull/111076 enables - // unstable cargo feature (`public-dependency`), we need to ensure - // that unstable features are enabled before reading libstd Cargo.toml. - cargo.env("RUSTC_BOOTSTRAP", "1"); - - for krate in &*self.crates { - cargo.arg("-p"); - cargo.arg(krate); - } - - builder.info(&format!( - "Cleaning{} stage{} {} artifacts ({} -> {})", - crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, - )); - - // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, - // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. - cargo.run(builder); - } - } - )+ } -} - -clean_crate_tree! { - Rustc, Mode::Rustc, "rustc-main"; - Std, Mode::Std, "sysroot"; -} - -fn clean(build: &Build, all: bool, stage: Option) { - if build.config.dry_run() { - return; - } - - rm_rf("tmp".as_ref()); - - // Clean the entire build directory - if all { - rm_rf(&build.out); - return; - } - - // Clean the target stage artifacts - if let Some(stage) = stage { - clean_specific_stage(build, stage); - return; - } - - // Follow the default behaviour - clean_default(build); -} - -fn clean_specific_stage(build: &Build, stage: u32) { - for host in &build.hosts { - let entries = match build.out.join(host).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - let stage_prefix = format!("stage{}", stage); - - // if current entry is not related with the target stage, continue - if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { - continue; - } - - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn clean_default(build: &Build) { - rm_rf(&build.out.join("tmp")); - rm_rf(&build.out.join("dist")); - rm_rf(&build.out.join("bootstrap").join(".last-warned-change-id")); - rm_rf(&build.out.join("bootstrap-shims-dump")); - rm_rf(&build.out.join("rustfmt.stamp")); - - let mut hosts: Vec<_> = build.hosts.iter().map(|t| build.out.join(t)).collect(); - // After cross-compilation, artifacts of the host architecture (which may differ from build.host) - // might not get removed. - // Adding its path (linked one for easier accessibility) will solve this problem. - hosts.push(build.out.join("host")); - - for host in hosts { - let entries = match host.read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - if entry.file_name().to_str() == Some("llvm") { - continue; - } - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn rm_rf(path: &Path) { - match path.symlink_metadata() { - Err(e) => { - if e.kind() == ErrorKind::NotFound { - return; - } - panic!("failed to get metadata for file {}: {}", path.display(), e); - } - Ok(metadata) => { - if metadata.file_type().is_file() || metadata.file_type().is_symlink() { - do_op(path, "remove file", |p| match fs::remove_file(p) { - #[cfg(windows)] - Err(e) - if e.kind() == std::io::ErrorKind::PermissionDenied - && p.file_name().and_then(std::ffi::OsStr::to_str) - == Some("bootstrap.exe") => - { - eprintln!("WARNING: failed to delete '{}'.", p.display()); - Ok(()) - } - r => r, - }); - - return; - } - - for file in t!(fs::read_dir(path)) { - rm_rf(&t!(file).path()); - } - - do_op(path, "remove dir", |p| match fs::remove_dir(p) { - // Check for dir not empty on Windows - // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, - // match on `e.kind()` instead. - #[cfg(windows)] - Err(e) if e.raw_os_error() == Some(145) => Ok(()), - r => r, - }); - } - }; -} - -fn do_op(path: &Path, desc: &str, mut f: F) -where - F: FnMut(&Path) -> io::Result<()>, -{ - match f(path) { - Ok(()) => {} - // On windows we can't remove a readonly file, and git will often clone files as readonly. - // As a result, we have some special logic to remove readonly files on windows. - // This is also the reason that we can't use things like fs::remove_dir_all(). - #[cfg(windows)] - Err(ref e) if e.kind() == ErrorKind::PermissionDenied => { - let m = t!(path.symlink_metadata()); - let mut p = m.permissions(); - // this os not unix, so clippy gives FP - #[expect(clippy::permissions_set_readonly_false)] - p.set_readonly(false); - t!(fs::set_permissions(path, p)); - f(path).unwrap_or_else(|e| { - // Delete symlinked directories on Windows - if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { - return; - } - panic!("failed to {} {}: {}", desc, path.display(), e); - }); - } - Err(e) => { - panic!("failed to {} {}: {}", desc, path.display(), e); - } - } -} diff --git a/standalonex/src/src/core/build_steps/clippy.rs b/standalonex/src/src/core/build_steps/clippy.rs deleted file mode 100644 index 0884d86c..00000000 --- a/standalonex/src/src/core/build_steps/clippy.rs +++ /dev/null @@ -1,409 +0,0 @@ -//! Implementation of running clippy on the compiler, standard library and various tools. - -use super::compile::{librustc_stamp, libstd_stamp, run_cargo, rustc_cargo, std_cargo}; -use super::tool::{SourceType, prepare_tool_cargo}; -use super::{check, compile}; -use crate::builder::{Builder, ShouldRun}; -use crate::core::build_steps::compile::std_crates_for_run_make; -use crate::core::builder; -use crate::core::builder::{Alias, Kind, RunConfig, Step, crate_description}; -use crate::{Mode, Subcommand, TargetSelection}; - -/// Disable the most spammy clippy lints -const IGNORED_RULES_FOR_STD_AND_RUSTC: &[&str] = &[ - "many_single_char_names", // there are a lot in stdarch - "collapsible_if", - "type_complexity", - "missing_safety_doc", // almost 3K warnings - "too_many_arguments", - "needless_lifetimes", // people want to keep the lifetimes - "wrong_self_convention", -]; - -fn lint_args(builder: &Builder<'_>, config: &LintConfig, ignored_rules: &[&str]) -> Vec { - fn strings<'a>(arr: &'a [&str]) -> impl Iterator + 'a { - arr.iter().copied().map(String::from) - } - - let Subcommand::Clippy { fix, allow_dirty, allow_staged, .. } = &builder.config.cmd else { - unreachable!("clippy::lint_args can only be called from `clippy` subcommands."); - }; - - let mut args = vec![]; - if *fix { - #[rustfmt::skip] - args.extend(strings(&[ - "--fix", "-Zunstable-options", - // FIXME: currently, `--fix` gives an error while checking tests for libtest, - // possibly because libtest is not yet built in the sysroot. - // As a workaround, avoid checking tests and benches when passed --fix. - "--lib", "--bins", "--examples", - ])); - - if *allow_dirty { - args.push("--allow-dirty".to_owned()); - } - - if *allow_staged { - args.push("--allow-staged".to_owned()); - } - } - - args.extend(strings(&["--"])); - - if config.deny.is_empty() && config.forbid.is_empty() { - args.extend(strings(&["--cap-lints", "warn"])); - } - - let all_args = std::env::args().collect::>(); - args.extend(get_clippy_rules_in_order(&all_args, config)); - - args.extend(ignored_rules.iter().map(|lint| format!("-Aclippy::{}", lint))); - args.extend(builder.config.free_args.clone()); - args -} - -/// We need to keep the order of the given clippy lint rules before passing them. -/// Since clap doesn't offer any useful interface for this purpose out of the box, -/// we have to handle it manually. -pub fn get_clippy_rules_in_order(all_args: &[String], config: &LintConfig) -> Vec { - let mut result = vec![]; - - for (prefix, item) in - [("-A", &config.allow), ("-D", &config.deny), ("-W", &config.warn), ("-F", &config.forbid)] - { - item.iter().for_each(|v| { - let rule = format!("{prefix}{v}"); - // Arguments added by bootstrap in LintConfig won't show up in the all_args list, so - // put them at the end of the command line. - let position = all_args.iter().position(|t| t == &rule || t == v).unwrap_or(usize::MAX); - result.push((position, rule)); - }); - } - - result.sort_by_key(|&(position, _)| position); - result.into_iter().map(|v| v.1).collect() -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct LintConfig { - pub allow: Vec, - pub warn: Vec, - pub deny: Vec, - pub forbid: Vec, -} - -impl LintConfig { - fn new(builder: &Builder<'_>) -> Self { - match builder.config.cmd.clone() { - Subcommand::Clippy { allow, deny, warn, forbid, .. } => { - Self { allow, warn, deny, forbid } - } - _ => unreachable!("LintConfig can only be called from `clippy` subcommands."), - } - } - - fn merge(&self, other: &Self) -> Self { - let merged = |self_attr: &[String], other_attr: &[String]| -> Vec { - self_attr.iter().cloned().chain(other_attr.iter().cloned()).collect() - }; - // This is written this way to ensure we get a compiler error if we add a new field. - Self { - allow: merged(&self.allow, &other.allow), - warn: merged(&self.warn, &other.warn), - deny: merged(&self.deny, &other.deny), - forbid: merged(&self.forbid, &other.forbid), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - config: LintConfig, - /// Whether to lint only a subset of crates. - crates: Vec, -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = std_crates_for_run_make(&run); - let config = LintConfig::new(run.builder); - run.builder.ensure(Std { target: run.target, config, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.require_submodule("library/stdarch", None); - - let target = self.target; - let compiler = builder.compiler(builder.top_stage, builder.config.build); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - Kind::Clippy, - ); - - std_cargo(builder, target, compiler.stage, &mut cargo); - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = - builder.msg_clippy(format_args!("library{}", crate_description(&self.crates)), target); - - run_cargo( - builder, - cargo, - lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC), - &libstd_stamp(builder, compiler, target), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - config: LintConfig, - /// Whether to lint only a subset of crates. - crates: Vec, -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Compiler); - let config = LintConfig::new(run.builder); - run.builder.ensure(Rustc { target: run.target, config, crates }); - } - - /// Lints the compiler. - /// - /// This will lint the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - if compiler.stage != 0 { - // If we're not in stage 0, then we won't have a std from the beta - // compiler around. That means we need to make sure there's one in - // the sysroot for the compiler to find. Otherwise, we're going to - // fail when building crates that need to generate code (e.g., build - // scripts and their dependencies). - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Std::new(compiler, target)); - } else { - builder.ensure(check::Std::new_with_build_kind(target, Some(Kind::Check))); - } - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Rustc, - SourceType::InTree, - target, - Kind::Clippy, - ); - - rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - - // Explicitly pass -p for all compiler crates -- this will force cargo - // to also lint the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = - builder.msg_clippy(format_args!("compiler{}", crate_description(&self.crates)), target); - - run_cargo( - builder, - cargo, - lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC), - &librustc_stamp(builder, compiler, target), - vec![], - true, - false, - ); - } -} - -macro_rules! lint_any { - ($( - $name:ident, $path:expr, $readable_name:expr - $(,lint_by_default = $lint_by_default:expr)* - ; - )+) => { - $( - - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub target: TargetSelection, - config: LintConfig, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = if false $(|| $lint_by_default)* { true } else { false }; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - let config = LintConfig::new(run.builder); - run.builder.ensure($name { - target: run.target, - config, - }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(check::Rustc::new_with_build_kind(target, builder, Some(Kind::Check))); - - let cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - Kind::Clippy, - $path, - SourceType::InTree, - &[], - ); - - let _guard = builder.msg_tool( - Kind::Clippy, - Mode::ToolRustc, - $readable_name, - compiler.stage, - &compiler.host, - &target, - ); - - let stamp = builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())); - - run_cargo( - builder, - cargo, - lint_args(builder, &self.config, &[]), - &stamp, - vec![], - true, - false, - ); - } - } - )+ - } -} - -lint_any!( - Bootstrap, "src/bootstrap", "bootstrap"; - BuildHelper, "src/build_helper", "build_helper"; - BuildManifest, "src/tools/build-manifest", "build-manifest"; - CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri"; - Clippy, "src/tools/clippy", "clippy"; - CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; - Compiletest, "src/tools/compiletest", "compiletest"; - CoverageDump, "src/tools/coverage-dump", "coverage-dump"; - Jsondocck, "src/tools/jsondocck", "jsondocck"; - Jsondoclint, "src/tools/jsondoclint", "jsondoclint"; - LintDocs, "src/tools/lint-docs", "lint-docs"; - LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker"; - Miri, "src/tools/miri", "miri"; - MiroptTestTools, "src/tools/miropt-test-tools", "miropt-test-tools"; - OptDist, "src/tools/opt-dist", "opt-dist"; - RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; - RemoteTestServer, "src/tools/remote-test-server", "remote-test-server"; - Rls, "src/tools/rls", "rls"; - RustAnalyzer, "src/tools/rust-analyzer", "rust-analyzer"; - Rustdoc, "src/librustdoc", "clippy"; - Rustfmt, "src/tools/rustfmt", "rustfmt"; - RustInstaller, "src/tools/rust-installer", "rust-installer"; - Tidy, "src/tools/tidy", "tidy"; - TestFloatParse, "src/etc/test-float-parse", "test-float-parse"; -); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CI { - target: TargetSelection, - config: LintConfig, -} - -impl Step for CI { - type Output = (); - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("ci") - } - - fn make_run(run: RunConfig<'_>) { - let config = LintConfig::new(run.builder); - run.builder.ensure(CI { target: run.target, config }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.ensure(Bootstrap { - target: self.target, - config: self.config.merge(&LintConfig { - allow: vec![], - warn: vec![], - deny: vec!["warnings".into()], - forbid: vec![], - }), - }); - let library_clippy_cfg = LintConfig { - allow: vec!["clippy::all".into()], - warn: vec![], - deny: vec!["clippy::correctness".into()], - forbid: vec![], - }; - let compiler_clippy_cfg = LintConfig { - allow: vec!["clippy::all".into()], - warn: vec![], - deny: vec!["clippy::correctness".into(), "clippy::clone_on_ref_ptr".into()], - forbid: vec![], - }; - - builder.ensure(Std { - target: self.target, - config: self.config.merge(&library_clippy_cfg), - crates: vec![], - }); - builder.ensure(Rustc { - target: self.target, - config: self.config.merge(&compiler_clippy_cfg), - crates: vec![], - }); - } -} diff --git a/standalonex/src/src/core/build_steps/compile.rs b/standalonex/src/src/core/build_steps/compile.rs deleted file mode 100644 index e99a0265..00000000 --- a/standalonex/src/src/core/build_steps/compile.rs +++ /dev/null @@ -1,2329 +0,0 @@ -//! Implementation of compiling various phases of the compiler and standard -//! library. -//! -//! This module contains some of the real meat in the bootstrap build system -//! which is where Cargo is used to compile the standard library, libtest, and -//! the compiler. This module is also responsible for assembling the sysroot as it -//! goes along from the output of the previous stage. - -use std::borrow::Cow; -use std::collections::HashSet; -use std::ffi::OsStr; -use std::io::BufReader; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::{env, fs, str}; - -use serde_derive::Deserialize; - -use crate::core::build_steps::tool::SourceType; -use crate::core::build_steps::{dist, llvm}; -use crate::core::builder; -use crate::core::builder::{ - Builder, Cargo, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath, crate_description, -}; -use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; -use crate::utils::exec::command; -use crate::utils::helpers::{ - exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, symlink_dir, t, up_to_date, -}; -use crate::{CLang, Compiler, DependencyType, GitRepo, LLVM_TOOLS, Mode}; - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Std { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build only a subset of crates in the standard library. - /// - /// This shouldn't be used from other steps; see the comment on [`Rustc`]. - crates: Vec, - /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, - /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overridden by `builder.ensure` from other steps. - force_recompile: bool, - extra_rust_args: &'static [&'static str], - is_for_mir_opt_tests: bool, -} - -impl Std { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - } - } - - pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: true, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - } - } - - pub fn new_for_mir_opt_tests(compiler: Compiler, target: TargetSelection) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args: &[], - is_for_mir_opt_tests: true, - } - } - - pub fn new_with_extra_rust_args( - compiler: Compiler, - target: TargetSelection, - extra_rust_args: &'static [&'static str], - ) -> Self { - Self { - target, - compiler, - crates: Default::default(), - force_recompile: false, - extra_rust_args, - is_for_mir_opt_tests: false, - } - } - - fn copy_extra_objects( - &self, - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, - ) -> Vec<(PathBuf, DependencyType)> { - let mut deps = Vec::new(); - if !self.is_for_mir_opt_tests { - deps.extend(copy_third_party_objects(builder, compiler, target)); - deps.extend(copy_self_contained_objects(builder, compiler, target)); - } - deps - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = std_crates_for_run_make(&run); - let builder = run.builder; - - // Force compilation of the standard library from source if the `library` is modified. This allows - // library team to compile the standard library without needing to compile the compiler with - // the `rust.download-rustc=true` option. - let force_recompile = builder.rust_info().is_managed_git_subrepository() - && builder.download_rustc() - && builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none(); - - run.builder.ensure(Std { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - force_recompile, - extra_rust_args: &[], - is_for_mir_opt_tests: false, - }); - } - - /// Builds the standard library. - /// - /// This will build the standard library for a particular stage of the build - /// using the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - - // When using `download-rustc`, we already have artifacts for the host available. Don't - // recompile them. - if builder.download_rustc() && target == builder.build.build - // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so - // its artifacts can't be reused. - && compiler.stage != 0 - && !self.force_recompile - { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - cp_rustc_component_to_ci_sysroot( - builder, - &sysroot, - builder.config.ci_rust_std_contents(), - ); - return; - } - - if builder.config.keep_stage.contains(&compiler.stage) - || builder.config.keep_stage_std.contains(&compiler.stage) - { - builder.info("WARNING: Using a potentially old libstd. This may not behave well."); - - builder.ensure(StartupObjects { compiler, target }); - - self.copy_extra_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler)); - return; - } - - builder.require_submodule("library/stdarch", None); - - let mut target_deps = builder.ensure(StartupObjects { compiler, target }); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Std::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting library (stage{} -> stage{})", - compiler_to_use.stage, compiler.stage - ) - } else { - format!( - "Uplifting library (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, compiler_to_use.host, compiler.stage, target - ) - }; - builder.info(&msg); - - // Even if we're not building std this stage, the new sysroot must - // still contain the third party objects needed by various targets. - self.copy_extra_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler_to_use)); - return; - } - - target_deps.extend(self.copy_extra_objects(builder, &compiler, target)); - - // The LLD wrappers and `rust-lld` are self-contained linking components that can be - // necessary to link the stdlib on some targets. We'll also need to copy these binaries to - // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. - if compiler.stage == 0 && compiler.host == builder.config.build { - // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. - let src_sysroot_bin = builder - .rustc_snapshot_sysroot() - .join("lib") - .join("rustlib") - .join(compiler.host) - .join("bin"); - if src_sysroot_bin.exists() { - let target_sysroot_bin = builder.sysroot_target_bindir(compiler, target); - t!(fs::create_dir_all(&target_sysroot_bin)); - builder.cp_link_r(&src_sysroot_bin, &target_sysroot_bin); - } - } - - // We build a sysroot for mir-opt tests using the same trick that Miri does: A check build - // with -Zalways-encode-mir. This frees us from the need to have a target linker, and the - // fact that this is a check build integrates nicely with run_cargo. - let mut cargo = if self.is_for_mir_opt_tests { - let mut cargo = builder::Cargo::new_for_mir_opt_tests( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - Kind::Check, - ); - cargo.rustflag("-Zalways-encode-mir"); - cargo.arg("--manifest-path").arg(builder.src.join("library/sysroot/Cargo.toml")); - cargo - } else { - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Std, - SourceType::InTree, - target, - Kind::Build, - ); - std_cargo(builder, target, compiler.stage, &mut cargo); - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - cargo - }; - - // See src/bootstrap/synthetic_targets.rs - if target.is_synthetic() { - cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); - } - for rustflag in self.extra_rust_args.iter() { - cargo.rustflag(rustflag); - } - - let _guard = builder.msg( - Kind::Build, - compiler.stage, - format_args!("library artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - run_cargo( - builder, - cargo, - vec![], - &libstd_stamp(builder, compiler, target), - target_deps, - self.is_for_mir_opt_tests, // is_check - false, - ); - - builder.ensure(StdLink::from_std( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -fn copy_and_stamp( - builder: &Builder<'_>, - libdir: &Path, - sourcedir: &Path, - name: &str, - target_deps: &mut Vec<(PathBuf, DependencyType)>, - dependency_type: DependencyType, -) { - let target = libdir.join(name); - builder.copy_link(&sourcedir.join(name), &target); - - target_deps.push((target, dependency_type)); -} - -fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { - let libunwind_path = builder.ensure(llvm::Libunwind { target }); - let libunwind_source = libunwind_path.join("libunwind.a"); - let libunwind_target = libdir.join("libunwind.a"); - builder.copy_link(&libunwind_source, &libunwind_target); - libunwind_target -} - -/// Copies third party objects needed by various targets. -fn copy_third_party_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let mut target_deps = vec![]; - - if builder.config.needs_sanitizer_runtime_built(target) && compiler.stage != 0 { - // The sanitizers are only copied in stage1 or above, - // to avoid creating dependency on LLVM. - target_deps.extend( - copy_sanitizers(builder, compiler, target) - .into_iter() - .map(|d| (d, DependencyType::Target)), - ); - } - - if target == "x86_64-fortanix-unknown-sgx" - || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree - && (target.contains("linux") || target.contains("fuchsia")) - { - let libunwind_path = - copy_llvm_libunwind(builder, target, &builder.sysroot_target_libdir(*compiler, target)); - target_deps.push((libunwind_path, DependencyType::Target)); - } - - target_deps -} - -/// Copies third party objects needed by various targets for self-contained linkage. -fn copy_self_contained_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let libdir_self_contained = - builder.sysroot_target_libdir(*compiler, target).join("self-contained"); - t!(fs::create_dir_all(&libdir_self_contained)); - let mut target_deps = vec![]; - - // Copies the libc and CRT objects. - // - // rustc historically provides a more self-contained installation for musl targets - // not requiring the presence of a native musl toolchain. For example, it can fall back - // to using gcc from a glibc-targeting toolchain for linking. - // To do that we have to distribute musl startup objects as a part of Rust toolchain - // and link with them manually in the self-contained mode. - if target.contains("musl") && !target.contains("unikraft") { - let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { - panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) - }); - for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); - for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { - let src = crt_path.join(obj); - let target = libdir_self_contained.join(obj); - builder.copy_link(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - - if !target.starts_with("s390x") { - let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); - target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); - } - } else if target.contains("-wasi") { - let srcdir = builder.wasi_libdir(target).unwrap_or_else(|| { - panic!( - "Target {:?} does not have a \"wasi-root\" key in Config.toml \ - or `$WASI_SDK_PATH` set", - target.triple - ) - }); - for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - } else if target.is_windows_gnu() { - for obj in ["crt2.o", "dllcrt2.o"].iter() { - let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); - let target = libdir_self_contained.join(obj); - builder.copy_link(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - } - - target_deps -} - -/// Resolves standard library crates for `Std::run_make` for any build kind (like check, build, clippy, etc.). -pub fn std_crates_for_run_make(run: &RunConfig<'_>) -> Vec { - // FIXME: Extend builder tests to cover the `crates` field of `Std` instances. - if cfg!(feature = "bootstrap-self-test") { - return vec![]; - } - - let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); - let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false); - - // For no_std targets, do not add any additional crates to the compilation other than what `compile::std_cargo` already adds for no_std targets. - if target_is_no_std { - vec![] - } - // If the paths include "library", build the entire standard library. - else if has_alias { - run.make_run_crates(builder::Alias::Library) - } else { - run.cargo_crates_in_set() - } -} - -/// Tries to find LLVM's `compiler-rt` source directory, for building `library/profiler_builtins`. -/// -/// Normally it lives in the `src/llvm-project` submodule, but if we will be using a -/// downloaded copy of CI LLVM, then we try to use the `compiler-rt` sources from -/// there instead, which lets us avoid checking out the LLVM submodule. -fn compiler_rt_for_profiler(builder: &Builder<'_>) -> PathBuf { - // Try to use `compiler-rt` sources from downloaded CI LLVM, if possible. - if builder.config.llvm_from_ci { - // CI LLVM might not have been downloaded yet, so try to download it now. - builder.config.maybe_download_ci_llvm(); - let ci_llvm_compiler_rt = builder.config.ci_llvm_root().join("compiler-rt"); - if ci_llvm_compiler_rt.exists() { - return ci_llvm_compiler_rt; - } - } - - // Otherwise, fall back to requiring the LLVM submodule. - builder.require_submodule("src/llvm-project", { - Some("The `build.profiler` config option requires `compiler-rt` sources from LLVM.") - }); - builder.src.join("src/llvm-project/compiler-rt") -} - -/// Configure cargo to compile the standard library, adding appropriate env vars -/// and such. -pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { - if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { - cargo.env("MACOSX_DEPLOYMENT_TARGET", target); - } - - // Paths needed by `library/profiler_builtins/build.rs`. - if let Some(path) = builder.config.profiler_path(target) { - cargo.env("LLVM_PROFILER_RT_LIB", path); - } else if builder.config.profiler_enabled(target) { - let compiler_rt = compiler_rt_for_profiler(builder); - // Currently this is separate from the env var used by `compiler_builtins` - // (below) so that adding support for CI LLVM here doesn't risk breaking - // the compiler builtins. But they could be unified if desired. - cargo.env("RUST_COMPILER_RT_FOR_PROFILER", compiler_rt); - } - - // Determine if we're going to compile in optimized C intrinsics to - // the `compiler-builtins` crate. These intrinsics live in LLVM's - // `compiler-rt` repository. - // - // Note that this shouldn't affect the correctness of `compiler-builtins`, - // but only its speed. Some intrinsics in C haven't been translated to Rust - // yet but that's pretty rare. Other intrinsics have optimized - // implementations in C which have only had slower versions ported to Rust, - // so we favor the C version where we can, but it's not critical. - // - // If `compiler-rt` is available ensure that the `c` feature of the - // `compiler-builtins` crate is enabled and it's configured to learn where - // `compiler-rt` is located. - let compiler_builtins_c_feature = if builder.config.optimized_compiler_builtins { - // NOTE: this interacts strangely with `llvm-has-rust-patches`. In that case, we enforce `submodules = false`, so this is a no-op. - // But, the user could still decide to manually use an in-tree submodule. - // - // NOTE: if we're using system llvm, we'll end up building a version of `compiler-rt` that doesn't match the LLVM we're linking to. - // That's probably ok? At least, the difference wasn't enforced before. There's a comment in - // the compiler_builtins build script that makes me nervous, though: - // https://github.com/rust-lang/compiler-builtins/blob/31ee4544dbe47903ce771270d6e3bea8654e9e50/build.rs#L575-L579 - builder.require_submodule( - "src/llvm-project", - Some( - "The `build.optimized-compiler-builtins` config option \ - requires `compiler-rt` sources from LLVM.", - ), - ); - let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); - assert!(compiler_builtins_root.exists()); - // The path to `compiler-rt` is also used by `profiler_builtins` (above), - // so if you're changing something here please also change that as appropriate. - cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); - " compiler-builtins-c" - } else { - "" - }; - - // `libtest` uses this to know whether or not to support - // `-Zunstable-options`. - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - let mut features = String::new(); - - if builder.no_std(target) == Some(true) { - features += " compiler-builtins-mem"; - if !target.starts_with("sbf") && !target.starts_with("bpf") { - features.push_str(compiler_builtins_c_feature); - } - - // for no-std targets we only compile a few no_std crates - cargo - .args(["-p", "alloc"]) - .arg("--manifest-path") - .arg(builder.src.join("library/alloc/Cargo.toml")) - .arg("--features") - .arg(features); - } else { - features += &builder.std_features(target); - features.push_str(compiler_builtins_c_feature); - - cargo - .arg("--features") - .arg(features) - .arg("--manifest-path") - .arg(builder.src.join("library/sysroot/Cargo.toml")); - - // Help the libc crate compile by assisting it in finding various - // sysroot native libraries. - if target.contains("musl") { - if let Some(p) = builder.musl_libdir(target) { - let root = format!("native={}", p.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - - if target.contains("-wasi") { - if let Some(dir) = builder.wasi_libdir(target) { - let root = format!("native={}", dir.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - } - - // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that - // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be - // built with bitcode so that the produced rlibs can be used for both LTO - // builds (which use bitcode) and non-LTO builds (which use object code). - // So we override the override here! - // - // But we don't bother for the stage 0 compiler because it's never used - // with LTO. - if stage >= 1 { - cargo.rustflag("-Cembed-bitcode=yes"); - } - if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // By default, rustc does not include unwind tables unless they are required - // for a particular target. They are not required by RISC-V targets, but - // compiling the standard library with them means that users can get - // backtraces without having to recompile the standard library themselves. - // - // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 - if target.contains("riscv") { - cargo.rustflag("-Cforce-unwind-tables=yes"); - } - - // Enable frame pointers by default for the library. Note that they are still controlled by a - // separate setting for the compiler. - cargo.rustflag("-Cforce-frame-pointers=yes"); - - let html_root = - format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); - cargo.rustflag(&html_root); - cargo.rustdocflag(&html_root); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct StdLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Vec, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl StdLink { - fn from_std(std: Std, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: std.compiler, - target: std.target, - crates: std.crates, - force_recompile: std.force_recompile, - } - } -} - -impl Step for StdLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Link all libstd rlibs/dylibs into the sysroot location. - /// - /// Links those artifacts generated by `compiler` to the `stage` compiler's - /// sysroot for the specified `host` and `target`. - /// - /// Note that this assumes that `compiler` has already generated the libstd - /// libraries for `target`, and this method will find them in the relevant - /// output directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - - // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. - let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { - // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { - compiler: self.compiler, - force_recompile: self.force_recompile, - }); - let libdir = sysroot.join(lib).join("rustlib").join(target).join("lib"); - let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host).join("lib"); - (libdir, hostdir) - } else { - let libdir = builder.sysroot_target_libdir(target_compiler, target); - let hostdir = builder.sysroot_target_libdir(target_compiler, compiler.host); - (libdir, hostdir) - }; - - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - - // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` - // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, - // and is not set to a custom path. - if compiler.stage == 0 - && builder - .build - .config - .initial_rustc - .starts_with(builder.out.join(compiler.host).join("stage0/bin")) - { - // Copy bin files from stage0/bin to stage0-sysroot/bin - let sysroot = builder.out.join(compiler.host).join("stage0-sysroot"); - - let host = compiler.host; - let stage0_bin_dir = builder.out.join(host).join("stage0/bin"); - let sysroot_bin_dir = sysroot.join("bin"); - t!(fs::create_dir_all(&sysroot_bin_dir)); - builder.cp_link_r(&stage0_bin_dir, &sysroot_bin_dir); - - // Copy all files from stage0/lib to stage0-sysroot/lib - let stage0_lib_dir = builder.out.join(host).join("stage0/lib"); - if let Ok(files) = fs::read_dir(stage0_lib_dir) { - for file in files { - let file = t!(file); - let path = file.path(); - if path.is_file() { - builder - .copy_link(&path, &sysroot.join("lib").join(path.file_name().unwrap())); - } - } - } - - // Copy codegen-backends from stage0 - let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); - t!(fs::create_dir_all(&sysroot_codegen_backends)); - let stage0_codegen_backends = builder - .out - .join(host) - .join("stage0/lib/rustlib") - .join(host) - .join("codegen-backends"); - if stage0_codegen_backends.exists() { - builder.cp_link_r(&stage0_codegen_backends, &sysroot_codegen_backends); - } - } - } -} - -/// Copies sanitizer runtime libraries into target libdir. -fn copy_sanitizers( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec { - let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); - - if builder.config.dry_run() { - return Vec::new(); - } - - let mut target_deps = Vec::new(); - let libdir = builder.sysroot_target_libdir(*compiler, target); - - for runtime in &runtimes { - let dst = libdir.join(&runtime.name); - builder.copy_link(&runtime.path, &dst); - - // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for - // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do - // not list them here to rename and sign the runtime library. - if target == "x86_64-apple-darwin" - || target == "aarch64-apple-darwin" - || target == "aarch64-apple-ios" - || target == "aarch64-apple-ios-sim" - || target == "x86_64-apple-ios" - { - // Update the library’s install name to reflect that it has been renamed. - apple_darwin_update_library_name(builder, &dst, &format!("@rpath/{}", runtime.name)); - // Upon renaming the install name, the code signature of the file will invalidate, - // so we will sign it again. - apple_darwin_sign_file(builder, &dst); - } - - target_deps.push(dst); - } - - target_deps -} - -fn apple_darwin_update_library_name(builder: &Builder<'_>, library_path: &Path, new_name: &str) { - command("install_name_tool").arg("-id").arg(new_name).arg(library_path).run(builder); -} - -fn apple_darwin_sign_file(builder: &Builder<'_>, file_path: &Path) { - command("codesign") - .arg("-f") // Force to rewrite the existing signature - .arg("-s") - .arg("-") - .arg(file_path) - .run(builder); -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct StartupObjects { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for StartupObjects { - type Output = Vec<(PathBuf, DependencyType)>; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("library/rtstartup") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(StartupObjects { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - }); - } - - /// Builds and prepare startup objects like rsbegin.o and rsend.o - /// - /// These are primarily used on Windows right now for linking executables/dlls. - /// They don't require any library support as they're just plain old object - /// files, so we just use the nightly snapshot compiler to always build them (as - /// no other compilers are guaranteed to be available). - fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { - let for_compiler = self.compiler; - let target = self.target; - if !target.is_windows_gnu() { - return vec![]; - } - - let mut target_deps = vec![]; - - let src_dir = &builder.src.join("library").join("rtstartup"); - let dst_dir = &builder.native_dir(target).join("rtstartup"); - let sysroot_dir = &builder.sysroot_target_libdir(for_compiler, target); - t!(fs::create_dir_all(dst_dir)); - - for file in &["rsbegin", "rsend"] { - let src_file = &src_dir.join(file.to_string() + ".rs"); - let dst_file = &dst_dir.join(file.to_string() + ".o"); - if !up_to_date(src_file, dst_file) { - let mut cmd = command(&builder.initial_rustc); - cmd.env("RUSTC_BOOTSTRAP", "1"); - if !builder.local_rebuild { - // a local_rebuild compiler already has stage1 features - cmd.arg("--cfg").arg("bootstrap"); - } - cmd.arg("--target") - .arg(target.rustc_target_arg()) - .arg("--emit=obj") - .arg("-o") - .arg(dst_file) - .arg(src_file) - .run(builder); - } - - let target = sysroot_dir.join((*file).to_string() + ".o"); - builder.copy_link(dst_file, &target); - target_deps.push((target, DependencyType::Target)); - } - - target_deps - } -} - -fn cp_rustc_component_to_ci_sysroot(builder: &Builder<'_>, sysroot: &Path, contents: Vec) { - let ci_rustc_dir = builder.config.ci_rustc_dir(); - - for file in contents { - let src = ci_rustc_dir.join(&file); - let dst = sysroot.join(file); - if src.is_dir() { - t!(fs::create_dir_all(dst)); - } else { - builder.copy_link(&src, &dst); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// The **previous** compiler used to compile this compiler. - pub compiler: Compiler, - /// Whether to build a subset of crates, rather than the whole compiler. - /// - /// This should only be requested by the user, not used within bootstrap itself. - /// Using it within bootstrap can lead to confusing situation where lints are replayed - /// in two different steps. - crates: Vec, -} - -impl Rustc { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default() } - } -} - -impl Step for Rustc { - /// We return the stage of the "actual" compiler (not the uplifted one). - /// - /// By "actual" we refer to the uplifting logic where we may not compile the requested stage; - /// instead, we uplift it from the previous stages. Which can lead to bootstrap failures in - /// specific situations where we request stage X from other steps. However we may end up - /// uplifting it from stage Y, causing the other stage to fail when attempting to link with - /// stage X which was never actually built. - type Output = u32; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let mut crates = run.builder.in_tree_crates("rustc-main", None); - for (i, krate) in crates.iter().enumerate() { - // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. - // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. - if krate.name == "rustc-main" { - crates.swap_remove(i); - break; - } - } - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.cargo_crates_in_set(); - run.builder.ensure(Rustc { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) -> u32 { - let compiler = self.compiler; - let target = self.target; - - // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, - // so its artifacts can't be reused. - if builder.download_rustc() && compiler.stage != 0 { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - cp_rustc_component_to_ci_sysroot( - builder, - &sysroot, - builder.config.ci_rustc_dev_contents(), - ); - return compiler.stage; - } - - builder.ensure(Std::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info("WARNING: Using a potentially old librustc. This may not behave well."); - builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self, compiler)); - - return compiler.stage; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Rustc::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting rustc (stage{} -> stage{})", - compiler_to_use.stage, - compiler.stage + 1 - ) - } else { - format!( - "Uplifting rustc (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, - compiler_to_use.host, - compiler.stage + 1, - target - ) - }; - builder.info(&msg); - builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); - return compiler_to_use.stage; - } - - // Ensure that build scripts and proc macros have a std / libproc_macro to link against. - builder.ensure(Std::new( - builder.compiler(self.compiler.stage, builder.config.build), - builder.config.build, - )); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Rustc, - SourceType::InTree, - target, - Kind::Build, - ); - - rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - - // NB: all RUSTFLAGS should be added to `rustc_cargo()` so they will be - // consistently applied by check/doc/test modes too. - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - if builder.build.config.enable_bolt_settings && compiler.stage == 1 { - // Relocations are required for BOLT to work. - cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); - } - - let _guard = builder.msg_sysroot_tool( - Kind::Build, - compiler.stage, - format_args!("compiler artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - let stamp = librustc_stamp(builder, compiler, target); - run_cargo( - builder, - cargo, - vec![], - &stamp, - vec![], - false, - true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. - ); - - // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain - // unexpected debuginfo from dependencies, for example from the C++ standard library used in - // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with - // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo - // away after the fact. - if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None - && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None - { - let target_root_dir = stamp.parent().unwrap(); - let rustc_driver = target_root_dir.join("librustc_driver.so"); - strip_debug(builder, target, &rustc_driver); - } - - builder.ensure(RustcLink::from_rustc( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - - compiler.stage - } -} - -pub fn rustc_cargo( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - compiler: &Compiler, - crates: &[String], -) { - cargo - .arg("--features") - .arg(builder.rustc_features(builder.kind, target, crates)) - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc/Cargo.toml")); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); - - // If the rustc output is piped to e.g. `head -n1` we want the process to be killed, rather than - // having an error bubble up and cause a panic. - // - // FIXME(jieyouxu): this flag is load-bearing for rustc to not ICE on broken pipes, because - // rustc internally sometimes uses std `println!` -- but std `println!` by default will panic on - // broken pipes, and uncaught panics will manifest as an ICE. The compiler *should* handle this - // properly, but this flag is set in the meantime to paper over the I/O errors. - // - // See for details. - // - // Also see the discussion for properly handling I/O errors related to broken pipes, i.e. safe - // variants of `println!` in - // . - cargo.rustflag("-Zon-broken-pipe=kill"); - - if builder.config.llvm_enzyme { - cargo.rustflag("-l").rustflag("Enzyme-19"); - } - - // Building with protected visibility reduces the number of dynamic relocations needed, giving - // us a faster startup time. However GNU ld < 2.40 will error if we try to link a shared object - // with direct references to protected symbols, so for now we only use protected symbols if - // linking with LLD is enabled. - if builder.build.config.lld_mode.is_used() { - cargo.rustflag("-Zdefault-visibility=protected"); - } - - // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary - // and may just be a time sink. - if compiler.stage != 0 { - match builder.config.rust_lto { - RustcLto::Thin | RustcLto::Fat => { - // Since using LTO for optimizing dylibs is currently experimental, - // we need to pass -Zdylib-lto. - cargo.rustflag("-Zdylib-lto"); - // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when - // compiling dylibs (and their dependencies), even when LTO is enabled for the - // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. - let lto_type = match builder.config.rust_lto { - RustcLto::Thin => "thin", - RustcLto::Fat => "fat", - _ => unreachable!(), - }; - cargo.rustflag(&format!("-Clto={lto_type}")); - cargo.rustflag("-Cembed-bitcode=yes"); - } - RustcLto::ThinLocal => { /* Do nothing, this is the default */ } - RustcLto::Off => { - cargo.rustflag("-Clto=off"); - } - } - } else if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // With LLD, we can use ICF (identical code folding) to reduce the executable size - // of librustc_driver/rustc and to improve i-cache utilization. - // - // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) - // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: - // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 - // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 - if builder.config.lld_mode.is_used() && !compiler.host.is_msvc() { - cargo.rustflag("-Clink-args=-Wl,--icf=all"); - } - - if builder.config.rust_profile_use.is_some() && builder.config.rust_profile_generate.is_some() { - panic!("Cannot use and generate PGO profiles at the same time"); - } - let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-generate={path}")); - // Apparently necessary to avoid overflowing the counters during - // a Cargo build profile - cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); - true - } else { - false - } - } else if let Some(path) = &builder.config.rust_profile_use { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-use={path}")); - if builder.is_verbose() { - cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); - } - true - } else { - false - } - } else { - false - }; - if is_collecting { - // Ensure paths to Rust sources are relative, not absolute. - cargo.rustflag(&format!( - "-Cllvm-args=-static-func-strip-dirname-prefix={}", - builder.config.src.components().count() - )); - } - - rustc_cargo_env(builder, cargo, target, compiler.stage); -} - -pub fn rustc_cargo_env( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - stage: u32, -) { - // Set some configuration variables picked up by build scripts and - // the compiler alike - cargo - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("CFG_VERSION", builder.rust_version()); - - // Some tools like Cargo detect their own git information in build scripts. When omit-git-hash - // is enabled in config.toml, we pass this environment variable to tell build scripts to avoid - // detecting git information on their own. - if builder.config.omit_git_hash { - cargo.env("CFG_OMIT_GIT_HASH", "1"); - } - - if let Some(backend) = builder.config.default_codegen_backend(target) { - cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); - } - - let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); - let target_config = builder.config.target_config.get(&target); - - cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); - - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - // Prefer the current target's own default_linker, else a globally - // specified one. - if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { - cargo.env("CFG_DEFAULT_LINKER", s); - } else if let Some(ref s) = builder.config.rustc_default_linker { - cargo.env("CFG_DEFAULT_LINKER", s); - } - - // Enable rustc's env var for `rust-lld` when requested. - if builder.config.lld_enabled - && (builder.config.channel == "dev" || builder.config.channel == "nightly") - { - cargo.env("CFG_USE_SELF_CONTAINED_LINKER", "1"); - } - - if builder.config.rust_verify_llvm_ir { - cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); - } - - if builder.config.llvm_enzyme { - cargo.rustflag("--cfg=llvm_enzyme"); - } - - // Note that this is disabled if LLVM itself is disabled or we're in a check - // build. If we are in a check build we still go ahead here presuming we've - // detected that LLVM is already built and good to go which helps prevent - // busting caches (e.g. like #71152). - if builder.config.llvm_enabled(target) { - let building_is_expensive = - crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target, false) - .should_build(); - // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler - let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; - let should_skip_build = building_is_expensive && can_skip_build; - if !should_skip_build { - rustc_llvm_env(builder, cargo, target) - } - } -} - -/// Pass down configuration from the LLVM build into the build of -/// rustc_llvm and rustc_codegen_llvm. -fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { - if builder.is_rust_llvm(target) { - cargo.env("LLVM_RUSTLLVM", "1"); - } - let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - cargo.env("LLVM_CONFIG", &llvm_config); - - // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script - // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by - // whitespace. - // - // For example: - // - on windows, when `clang-cl` is used with instrumentation, we need to manually add - // clang's runtime library resource directory so that the profiler runtime library can be - // found. This is to avoid the linker errors about undefined references to - // `__llvm_profile_instrument_memop` when linking `rustc_driver`. - let mut llvm_linker_flags = String::new(); - if builder.config.llvm_profile_generate && target.is_msvc() { - if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { - // Add clang's runtime library directory to the search path - let clang_rt_dir = get_clang_cl_resource_dir(builder, clang_cl_path); - llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); - } - } - - // The config can also specify its own llvm linker flags. - if let Some(ref s) = builder.config.llvm_ldflags { - if !llvm_linker_flags.is_empty() { - llvm_linker_flags.push(' '); - } - llvm_linker_flags.push_str(s); - } - - // Set the linker flags via the env var that `rustc_llvm`'s build script will read. - if !llvm_linker_flags.is_empty() { - cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); - } - - // Building with a static libstdc++ is only supported on linux right now, - // not for MSVC or macOS - if builder.config.llvm_static_stdcpp - && !target.contains("freebsd") - && !target.is_msvc() - && !target.contains("apple") - && !target.contains("solaris") - { - let file = compiler_file( - builder, - &builder.cxx(target).unwrap(), - target, - CLang::Cxx, - "libstdc++.a", - ); - cargo.env("LLVM_STATIC_STDCPP", file); - } - if builder.llvm_link_shared() { - cargo.env("LLVM_LINK_SHARED", "1"); - } - if builder.config.llvm_use_libcxx { - cargo.env("LLVM_USE_LIBCXX", "1"); - } - if builder.config.llvm_assertions { - cargo.env("LLVM_ASSERTIONS", "1"); - } -} - -/// `RustcLink` copies all of the rlibs from the rustc build into the previous stage's sysroot. -/// This is necessary for tools using `rustc_private`, where the previous compiler will build -/// a tool against the next compiler. -/// To build a tool against a compiler, the rlibs of that compiler that it links against -/// must be in the sysroot of the compiler that's doing the compiling. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RustcLink { - /// The compiler whose rlibs we are copying around. - pub compiler: Compiler, - /// This is the compiler into whose sysroot we want to copy the rlibs into. - pub previous_stage_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Vec, -} - -impl RustcLink { - fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - previous_stage_compiler: rustc.compiler, - target: rustc.target, - crates: rustc.crates, - } - } -} - -impl Step for RustcLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Same as `std_link`, only for librustc - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let previous_stage_compiler = self.previous_stage_compiler; - let target = self.target; - add_to_sysroot( - builder, - &builder.sysroot_target_libdir(previous_stage_compiler, target), - &builder.sysroot_target_libdir(previous_stage_compiler, compiler.host), - &librustc_stamp(builder, compiler, target), - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub compiler: Compiler, - pub backend: String, -} - -fn needs_codegen_config(run: &RunConfig<'_>) -> bool { - let mut needs_codegen_cfg = false; - for path_set in &run.paths { - needs_codegen_cfg = match path_set { - PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), - PathSet::Suite(suite) => is_codegen_cfg_needed(suite, run), - } - } - needs_codegen_cfg -} - -pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; - -fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { - let path = path.path.to_str().unwrap(); - - let is_explicitly_called = |p| -> bool { run.builder.paths.contains(p) }; - let should_enforce = run.builder.kind == Kind::Dist || run.builder.kind == Kind::Install; - - if path.contains(CODEGEN_BACKEND_PREFIX) { - let mut needs_codegen_backend_config = true; - for backend in run.builder.config.codegen_backends(run.target) { - if path.ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + backend)) { - needs_codegen_backend_config = false; - } - } - if (is_explicitly_called(&PathBuf::from(path)) || should_enforce) - && needs_codegen_backend_config - { - run.builder.info( - "WARNING: no codegen-backends config matched the requested path to build a codegen backend. \ - HELP: add backend to codegen-backends in config.toml.", - ); - return true; - } - } - - false -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - /// Only the backends specified in the `codegen-backends` entry of `config.toml` are built. - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - if needs_codegen_config(&run) { - return; - } - - for backend in run.builder.config.codegen_backends(run.target) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - run.builder.ensure(CodegenBackend { - target: run.target, - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - backend: backend.clone(), - }); - } - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info( - "WARNING: Using a potentially old codegen backend. \ - This may not behave well.", - ); - // Codegen backends are linked separately from this step today, so we don't do - // anything here. - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); - return; - } - - let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); - - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Codegen, - SourceType::InTree, - target, - Kind::Build, - ); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let tmp_stamp = out_dir.join(".tmp.stamp"); - - let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); - if builder.config.dry_run() { - return; - } - let mut files = files.into_iter().filter(|f| { - let filename = f.file_name().unwrap().to_str().unwrap(); - is_dylib(f) && filename.contains("rustc_codegen_") - }); - let codegen_backend = match files.next() { - Some(f) => f, - None => panic!("no dylibs built for codegen backend?"), - }; - if let Some(f) = files.next() { - panic!( - "codegen backend built two dylibs:\n{}\n{}", - codegen_backend.display(), - f.display() - ); - } - let stamp = codegen_backend_stamp(builder, compiler, target, &backend); - let codegen_backend = codegen_backend.to_str().unwrap(); - t!(fs::write(stamp, codegen_backend)); - } -} - -/// Creates the `codegen-backends` folder for a compiler that's about to be -/// assembled as a complete compiler. -/// -/// This will take the codegen artifacts produced by `compiler` and link them -/// into an appropriate location for `target_compiler` to be a functional -/// compiler. -fn copy_codegen_backends_to_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - target_compiler: Compiler, -) { - let target = target_compiler.host; - - // Note that this step is different than all the other `*Link` steps in - // that it's not assembling a bunch of libraries but rather is primarily - // moving the codegen backend into place. The codegen backend of rustc is - // not linked into the main compiler by default but is rather dynamically - // selected at runtime for inclusion. - // - // Here we're looking for the output dylib of the `CodegenBackend` step and - // we're copying that into the `codegen-backends` folder. - let dst = builder.sysroot_codegen_backends(target_compiler); - t!(fs::create_dir_all(&dst), dst); - - if builder.config.dry_run() { - return; - } - - for backend in builder.config.codegen_backends(target) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - let stamp = codegen_backend_stamp(builder, compiler, target, backend); - let dylib = t!(fs::read_to_string(&stamp)); - let file = Path::new(&dylib); - let filename = file.file_name().unwrap().to_str().unwrap(); - // change `librustc_codegen_cranelift-xxxxxx.so` to - // `librustc_codegen_cranelift-release.so` - let target_filename = { - let dash = filename.find('-').unwrap(); - let dot = filename.find('.').unwrap(); - format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) - }; - builder.copy_link(file, &dst.join(target_filename)); - } -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -pub fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: &str, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}.stamp")) -} - -pub fn compiler_file( - builder: &Builder<'_>, - compiler: &Path, - target: TargetSelection, - c: CLang, - file: &str, -) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::new(); - } - let mut cmd = command(compiler); - cmd.args(builder.cflags(target, GitRepo::Rustc, c)); - cmd.arg(format!("-print-file-name={file}")); - let out = cmd.run_capture_stdout(builder).stdout(); - PathBuf::from(out.trim()) -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Sysroot { - pub compiler: Compiler, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl Sysroot { - pub(crate) fn new(compiler: Compiler) -> Self { - Sysroot { compiler, force_recompile: false } - } -} - -impl Step for Sysroot { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Returns the sysroot that `compiler` is supposed to use. - /// For the stage0 compiler, this is stage0-sysroot (because of the initial std build). - /// For all other stages, it's the same stage directory that the compiler lives in. - fn run(self, builder: &Builder<'_>) -> PathBuf { - let compiler = self.compiler; - let host_dir = builder.out.join(compiler.host); - - let sysroot_dir = |stage| { - if stage == 0 { - host_dir.join("stage0-sysroot") - } else if self.force_recompile && stage == compiler.stage { - host_dir.join(format!("stage{stage}-test-sysroot")) - } else if builder.download_rustc() && compiler.stage != builder.top_stage { - host_dir.join("ci-rustc-sysroot") - } else { - host_dir.join(format!("stage{}", stage)) - } - }; - let sysroot = sysroot_dir(compiler.stage); - - builder - .verbose(|| println!("Removing sysroot {} to avoid caching bugs", sysroot.display())); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - - // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 - // compiler relies on more recent version of LLVM than the beta compiler, it may not - // be able to locate the correct LLVM in the sysroot. This situation typically occurs - // when we upgrade LLVM version while the beta compiler continues to use an older version. - // - // Make sure to add the correct version of LLVM into the stage0 sysroot. - if compiler.stage == 0 { - dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); - } - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() && compiler.stage != 0 { - assert_eq!( - builder.config.build, compiler.host, - "Cross-compiling is not yet supported with `download-rustc`", - ); - - // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. - for stage in 0..=2 { - if stage != compiler.stage { - let dir = sysroot_dir(stage); - if !dir.ends_with("ci-rustc-sysroot") { - let _ = fs::remove_dir_all(dir); - } - } - } - - // Copy the compiler into the correct sysroot. - // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. - // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. - // There are a few quirks of bootstrap that interact to make this reliable: - // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This - // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to - // fail because of duplicate metadata. - // 2. The sysroot is deleted and recreated between each invocation, so running `x test - // ui-fulldeps && x test ui` can't cause failures. - let mut filtered_files = Vec::new(); - let mut add_filtered_files = |suffix, contents| { - for path in contents { - let path = Path::new(&path); - if path.parent().map_or(false, |parent| parent.ends_with(suffix)) { - filtered_files.push(path.file_name().unwrap().to_owned()); - } - } - }; - let suffix = format!("lib/rustlib/{}/lib", compiler.host); - add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); - // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the - // newly compiled std, not the downloaded std. - add_filtered_files("lib", builder.config.ci_rust_std_contents()); - - let filtered_extensions = [ - OsStr::new("rmeta"), - OsStr::new("rlib"), - // FIXME: this is wrong when compiler.host != build, but we don't support that today - OsStr::new(std::env::consts::DLL_EXTENSION), - ]; - let ci_rustc_dir = builder.config.ci_rustc_dir(); - builder.cp_link_filtered(&ci_rustc_dir, &sysroot, &|path| { - if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { - return true; - } - if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { - return true; - } - if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { - builder.verbose_than(1, || println!("ignoring {}", path.display())); - false - } else { - true - } - }); - } - - // Symlink the source root into the same location inside the sysroot, - // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), - // so that any tools relying on `rust-src` also work for local builds, - // and also for translating the virtual `/rustc/$hash` back to the real - // directory (for running tests with `rust.remap-debuginfo = true`). - let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); - let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); - if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { - eprintln!( - "ERROR: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_src_rust.display(), - builder.src.display(), - e, - ); - if builder.config.rust_remap_debuginfo { - eprintln!( - "ERROR: some `tests/ui` tests will fail when lacking `{}`", - sysroot_lib_rustlib_src_rust.display(), - ); - } - build_helper::exit!(1); - } - - // rustc-src component is already part of CI rustc's sysroot - if !builder.download_rustc() { - let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); - let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); - if let Err(e) = - symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) - { - eprintln!( - "ERROR: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_rustcsrc_rust.display(), - builder.src.display(), - e, - ); - build_helper::exit!(1); - } - } - - sysroot - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Assemble { - /// The compiler which we will produce in this step. Assemble itself will - /// take care of ensuring that the necessary prerequisites to do so exist, - /// that is, this target can be a stage2 compiler and Assemble will build - /// previous stages for you. - pub target_compiler: Compiler, -} - -impl Step for Assemble { - type Output = Compiler; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Assemble { - target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), - }); - } - - /// Prepare a new compiler from the artifacts in `stage` - /// - /// This will assemble a compiler in `build/$host/stage$stage`. The compiler - /// must have been previously produced by the `stage - 1` builder.build - /// compiler. - fn run(self, builder: &Builder<'_>) -> Compiler { - let target_compiler = self.target_compiler; - - if target_compiler.stage == 0 { - assert_eq!( - builder.config.build, target_compiler.host, - "Cannot obtain compiler for non-native build triple at stage 0" - ); - // The stage 0 compiler for the build triple is always pre-built. - return target_compiler; - } - - // We prepend this bin directory to the user PATH when linking Rust binaries. To - // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. - let libdir = builder.sysroot_target_libdir(target_compiler, target_compiler.host); - let libdir_bin = libdir.parent().unwrap().join("bin"); - t!(fs::create_dir_all(&libdir_bin)); - - if builder.config.llvm_enabled(target_compiler.host) { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: target_compiler.host }); - if !builder.config.dry_run() && builder.config.llvm_tools_enabled { - let llvm_bin_dir = - command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout(); - let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); - - // Since we've already built the LLVM tools, install them to the sysroot. - // This is the equivalent of installing the `llvm-tools-preview` component via - // rustup, and lets developers use a locally built toolchain to - // build projects that expect llvm tools to be present in the sysroot - // (e.g. the `bootimage` crate). - for tool in LLVM_TOOLS { - let tool_exe = exe(tool, target_compiler.host); - let src_path = llvm_bin_dir.join(&tool_exe); - // When using `download-ci-llvm`, some of the tools - // may not exist, so skip trying to copy them. - if src_path.exists() { - // There is a chance that these tools are being installed from an external LLVM. - // Use `Builder::resolve_symlink_and_copy` instead of `Builder::copy_link` to ensure - // we are copying the original file not the symlinked path, which causes issues for - // tarball distribution. - // - // See https://github.com/rust-lang/rust/issues/135554. - builder.resolve_symlink_and_copy(&src_path, &libdir_bin.join(&tool_exe)); - } - } - } - } - - let maybe_install_llvm_bitcode_linker = |compiler| { - if builder.config.llvm_bitcode_linker_enabled { - let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { - compiler, - target: target_compiler.host, - extra_features: vec![], - }); - let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); - builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); - } - }; - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() { - builder.ensure(Std::new(target_compiler, target_compiler.host)); - let sysroot = - builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); - // Ensure that `libLLVM.so` ends up in the newly created target directory, - // so that tools using `rustc_private` can use it. - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - // Lower stages use `ci-rustc-sysroot`, not stageN - if target_compiler.stage == builder.top_stage { - builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); - } - - maybe_install_llvm_bitcode_linker(target_compiler); - - return target_compiler; - } - - // Get the compiler that we'll use to bootstrap ourselves. - // - // Note that this is where the recursive nature of the bootstrap - // happens, as this will request the previous stage's compiler on - // downwards to stage 0. - // - // Also note that we're building a compiler for the host platform. We - // only assume that we can run `build` artifacts, which means that to - // produce some other architecture compiler we need to start from - // `build` to get there. - // - // FIXME: It may be faster if we build just a stage 1 compiler and then - // use that to bootstrap this compiler forward. - let mut build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // Build enzyme - let enzyme_install = if builder.config.llvm_enzyme { - Some(builder.ensure(llvm::Enzyme { target: build_compiler.host })) - } else { - None - }; - - if let Some(enzyme_install) = enzyme_install { - let lib_ext = std::env::consts::DLL_EXTENSION; - let src_lib = enzyme_install.join("build/Enzyme/libEnzyme-19").with_extension(lib_ext); - let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); - let target_libdir = - builder.sysroot_target_libdir(target_compiler, target_compiler.host); - let dst_lib = libdir.join("libEnzyme-19").with_extension(lib_ext); - let target_dst_lib = target_libdir.join("libEnzyme-19").with_extension(lib_ext); - builder.copy_link(&src_lib, &dst_lib); - builder.copy_link(&src_lib, &target_dst_lib); - } - - // Build the libraries for this compiler to link to (i.e., the libraries - // it uses at runtime). NOTE: Crates the target compiler compiles don't - // link to these. (FIXME: Is that correct? It seems to be correct most - // of the time but I think we do link to these for stage2/bin compilers - // when not performing a full bootstrap). - let actual_stage = builder.ensure(Rustc::new(build_compiler, target_compiler.host)); - // Current build_compiler.stage might be uplifted instead of being built; so update it - // to not fail while linking the artifacts. - build_compiler.stage = actual_stage; - - for backend in builder.config.codegen_backends(target_compiler.host) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - builder.ensure(CodegenBackend { - compiler: build_compiler, - target: target_compiler.host, - backend: backend.clone(), - }); - } - - let lld_install = if builder.config.lld_enabled { - Some(builder.ensure(llvm::Lld { target: target_compiler.host })) - } else { - None - }; - - let stage = target_compiler.stage; - let host = target_compiler.host; - let (host_info, dir_name) = if build_compiler.host == host { - ("".into(), "host".into()) - } else { - (format!(" ({host})"), host.to_string()) - }; - // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since - // sysroots can temporarily be empty until we put the compiler inside. However, - // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any - // ambiguity. - let msg = format!( - "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" - ); - builder.info(&msg); - - // Link in all dylibs to the libdir - let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); - let proc_macros = builder - .read_stamp_file(&stamp) - .into_iter() - .filter_map(|(path, dependency_type)| { - if dependency_type == DependencyType::Host { - Some(path.file_name().unwrap().to_owned().into_string().unwrap()) - } else { - None - } - }) - .collect::>(); - - let sysroot = builder.sysroot(target_compiler); - let rustc_libdir = builder.rustc_libdir(target_compiler); - t!(fs::create_dir_all(&rustc_libdir)); - let src_libdir = builder.sysroot_target_libdir(build_compiler, host); - for f in builder.read_dir(&src_libdir) { - let filename = f.file_name().into_string().unwrap(); - - let is_proc_macro = proc_macros.contains(&filename); - let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename); - - // If we link statically to stdlib, do not copy the libstd dynamic library file - // FIXME: Also do this for Windows once incremental post-optimization stage0 tests - // work without std.dll (see https://github.com/rust-lang/rust/pull/131188). - let can_be_rustc_dynamic_dep = if builder - .link_std_into_rustc_driver(target_compiler.host) - && !target_compiler.host.is_windows() - { - let is_std = filename.starts_with("std-") || filename.starts_with("libstd-"); - !is_std - } else { - true - }; - - if is_dylib_or_debug && can_be_rustc_dynamic_dep && !is_proc_macro { - builder.copy_link(&f.path(), &rustc_libdir.join(&filename)); - } - } - - copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - - if let Some(lld_install) = lld_install { - let src_exe = exe("lld", target_compiler.host); - let dst_exe = exe("rust-lld", target_compiler.host); - builder.copy_link(&lld_install.join("bin").join(src_exe), &libdir_bin.join(dst_exe)); - let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir_all(&self_contained_lld_dir)); - let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { - compiler: build_compiler, - target: target_compiler.host, - }); - for name in crate::LLD_FILE_NAMES { - builder.copy_link( - &lld_wrapper_exe, - &self_contained_lld_dir.join(exe(name, target_compiler.host)), - ); - } - } - - if builder.config.llvm_enabled(target_compiler.host) && builder.config.llvm_tools_enabled { - // `llvm-strip` is used by rustc, which is actually just a symlink to `llvm-objcopy`, so - // copy and rename `llvm-objcopy`. - // - // But only do so if llvm-tools are enabled, as bootstrap compiler might not contain any - // LLVM tools, e.g. for cg_clif. - // See . - let src_exe = exe("llvm-objcopy", target_compiler.host); - let dst_exe = exe("rust-objcopy", target_compiler.host); - builder.copy_link(&libdir_bin.join(src_exe), &libdir_bin.join(dst_exe)); - } - - // In addition to `rust-lld` also install `wasm-component-ld` when - // LLD is enabled. This is a relatively small binary that primarily - // delegates to the `rust-lld` binary for linking and then runs - // logic to create the final binary. This is used by the - // `wasm32-wasip2` target of Rust. - if builder.tool_enabled("wasm-component-ld") { - let wasm_component_ld_exe = - builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, - target: target_compiler.host, - }); - builder.copy_link( - &wasm_component_ld_exe, - &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), - ); - } - - maybe_install_llvm_bitcode_linker(build_compiler); - - // Ensure that `libLLVM.so` ends up in the newly build compiler directory, - // so that it can be found when the newly built `rustc` is run. - dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - - // Link the compiler binary itself into place - let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); - let rustc = out_dir.join(exe("rustc-main", host)); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(bindir)); - let compiler = builder.rustc(target_compiler); - builder.copy_link(&rustc, &compiler); - - target_compiler - } -} - -/// Link some files into a rustc sysroot. -/// -/// For a particular stage this will link the file listed in `stamp` into the -/// `sysroot_dst` provided. -pub fn add_to_sysroot( - builder: &Builder<'_>, - sysroot_dst: &Path, - sysroot_host_dst: &Path, - stamp: &Path, -) { - let self_contained_dst = &sysroot_dst.join("self-contained"); - t!(fs::create_dir_all(sysroot_dst)); - t!(fs::create_dir_all(sysroot_host_dst)); - t!(fs::create_dir_all(self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - let dst = match dependency_type { - DependencyType::Host => sysroot_host_dst, - DependencyType::Target => sysroot_dst, - DependencyType::TargetSelfContained => self_contained_dst, - }; - builder.copy_link(&path, &dst.join(path.file_name().unwrap())); - } -} - -pub fn run_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - stamp: &Path, - additional_target_deps: Vec<(PathBuf, DependencyType)>, - is_check: bool, - rlib_only_metadata: bool, -) -> Vec { - // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.parent().unwrap(); - // `target_deps_dir` looks like $dir/$target/release/deps - let target_deps_dir = target_root_dir.join("deps"); - // `host_root_dir` looks like $dir/release - let host_root_dir = target_root_dir - .parent() - .unwrap() // chop off `release` - .parent() - .unwrap() // chop off `$target` - .join(target_root_dir.file_name().unwrap()); - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let mut deps = Vec::new(); - let mut toplevel = Vec::new(); - let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { - let (filenames, crate_types) = match msg { - CargoMessage::CompilerArtifact { - filenames, - target: CargoTarget { crate_types }, - .. - } => (filenames, crate_types), - _ => return, - }; - for filename in filenames { - // Skip files like executables - let mut keep = false; - if filename.ends_with(".lib") - || filename.ends_with(".a") - || is_debug_info(&filename) - || is_dylib(Path::new(&*filename)) - { - // Always keep native libraries, rust dylibs and debuginfo - keep = true; - } - if is_check && filename.ends_with(".rmeta") { - // During check builds we need to keep crate metadata - keep = true; - } else if rlib_only_metadata { - if filename.contains("jemalloc_sys") - || filename.contains("rustc_smir") - || filename.contains("stable_mir") - { - // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, - // so we need to distribute them as rlib to be able to use them. - keep |= filename.ends_with(".rlib"); - } else { - // Distribute the rest of the rustc crates as rmeta files only to reduce - // the tarball sizes by about 50%. The object files are linked into - // librustc_driver.so, so it is still possible to link against them. - keep |= filename.ends_with(".rmeta"); - } - } else { - // In all other cases keep all rlibs - keep |= filename.ends_with(".rlib"); - } - - if !keep { - continue; - } - - let filename = Path::new(&*filename); - - // If this was an output file in the "host dir" we don't actually - // worry about it, it's not relevant for us - if filename.starts_with(&host_root_dir) { - // Unless it's a proc macro used in the compiler - if crate_types.iter().any(|t| t == "proc-macro") { - deps.push((filename.to_path_buf(), DependencyType::Host)); - } - continue; - } - - // If this was output in the `deps` dir then this is a precise file - // name (hash included) so we start tracking it. - if filename.starts_with(&target_deps_dir) { - deps.push((filename.to_path_buf(), DependencyType::Target)); - continue; - } - - // Otherwise this was a "top level artifact" which right now doesn't - // have a hash in the name, but there's a version of this file in - // the `deps` folder which *does* have a hash in the name. That's - // the one we'll want to we'll probe for it later. - // - // We do not use `Path::file_stem` or `Path::extension` here, - // because some generated files may have multiple extensions e.g. - // `std-.dll.lib` on Windows. The aforementioned methods only - // split the file name by the last extension (`.lib`) while we need - // to split by all extensions (`.dll.lib`). - let expected_len = t!(filename.metadata()).len(); - let filename = filename.file_name().unwrap().to_str().unwrap(); - let mut parts = filename.splitn(2, '.'); - let file_stem = parts.next().unwrap().to_owned(); - let extension = parts.next().unwrap().to_owned(); - - toplevel.push((file_stem, extension, expected_len)); - } - }); - - if !ok { - crate::exit!(1); - } - - if builder.config.dry_run() { - return Vec::new(); - } - - // Ok now we need to actually find all the files listed in `toplevel`. We've - // got a list of prefix/extensions and we basically just need to find the - // most recent file in the `deps` folder corresponding to each one. - let contents = t!(target_deps_dir.read_dir()) - .map(|e| t!(e)) - .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) - .collect::>(); - for (prefix, extension, expected_len) in toplevel { - let candidates = contents.iter().filter(|&(_, filename, meta)| { - meta.len() == expected_len - && filename - .strip_prefix(&prefix[..]) - .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) - .unwrap_or(false) - }); - let max = candidates.max_by_key(|&(_, _, metadata)| { - metadata.modified().expect("mtime should be available on all relevant OSes") - }); - let path_to_add = match max { - Some(triple) => triple.0.to_str().unwrap(), - None => panic!("no output generated for {prefix:?} {extension:?}"), - }; - if is_dylib(Path::new(path_to_add)) { - let candidate = format!("{path_to_add}.lib"); - let candidate = PathBuf::from(candidate); - if candidate.exists() { - deps.push((candidate, DependencyType::Target)); - } - } - deps.push((path_to_add.into(), DependencyType::Target)); - } - - deps.extend(additional_target_deps); - deps.sort(); - let mut new_contents = Vec::new(); - for (dep, dependency_type) in deps.iter() { - new_contents.extend(match *dependency_type { - DependencyType::Host => b"h", - DependencyType::Target => b"t", - DependencyType::TargetSelfContained => b"s", - }); - new_contents.extend(dep.to_str().unwrap().as_bytes()); - new_contents.extend(b"\0"); - } - t!(fs::write(stamp, &new_contents)); - deps.into_iter().map(|(d, _)| d).collect() -} - -pub fn stream_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - cb: &mut dyn FnMut(CargoMessage<'_>), -) -> bool { - let mut cmd = cargo.into_cmd(); - let cargo = cmd.as_command_mut(); - // Instruct Cargo to give us json messages on stdout, critically leaving - // stderr as piped so we can get those pretty colors. - let mut message_format = if builder.config.json_output { - String::from("json") - } else { - String::from("json-render-diagnostics") - }; - if let Some(s) = &builder.config.rustc_error_format { - message_format.push_str(",json-diagnostic-"); - message_format.push_str(s); - } - cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); - - for arg in tail_args { - cargo.arg(arg); - } - - builder.verbose(|| println!("running: {cargo:?}")); - - if builder.config.dry_run() { - return true; - } - - let mut child = match cargo.spawn() { - Ok(child) => child, - Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), - }; - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let stdout = BufReader::new(child.stdout.take().unwrap()); - for line in stdout.lines() { - let line = t!(line); - match serde_json::from_str::>(&line) { - Ok(msg) => { - if builder.config.json_output { - // Forward JSON to stdout. - println!("{line}"); - } - cb(msg) - } - // If this was informational, just print it out and continue - Err(_) => println!("{line}"), - } - } - - // Make sure Cargo actually succeeded after we read all of its stdout. - let status = t!(child.wait()); - if builder.is_verbose() && !status.success() { - eprintln!( - "command did not execute successfully: {cargo:?}\n\ - expected success, got: {status}" - ); - } - status.success() -} - -#[derive(Deserialize)] -pub struct CargoTarget<'a> { - crate_types: Vec>, -} - -#[derive(Deserialize)] -#[serde(tag = "reason", rename_all = "kebab-case")] -pub enum CargoMessage<'a> { - CompilerArtifact { filenames: Vec>, target: CargoTarget<'a> }, - BuildScriptExecuted, - BuildFinished, -} - -pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { - // FIXME: to make things simpler for now, limit this to the host and target where we know - // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not - // cross-compiling. Expand this to other appropriate targets in the future. - if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { - return; - } - - let previous_mtime = t!(t!(path.metadata()).modified()); - command("strip").arg("--strip-debug").arg(path).run_capture(builder); - - let file = t!(fs::File::open(path)); - - // After running `strip`, we have to set the file modification time to what it was before, - // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time - // bootstrap is invoked. - // - // An example of this is if we run this on librustc_driver.so. In the first invocation: - // - Cargo will build librustc_driver.so (mtime of 1) - // - Cargo will build rustc-main (mtime of 2) - // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). - // - // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so - // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause - // everything else (standard library, future stages...) to be rebuilt. - t!(file.set_modified(previous_mtime)); -} diff --git a/standalonex/src/src/core/build_steps/dist.rs b/standalonex/src/src/core/build_steps/dist.rs deleted file mode 100644 index cb352e21..00000000 --- a/standalonex/src/src/core/build_steps/dist.rs +++ /dev/null @@ -1,2435 +0,0 @@ -//! Implementation of the various distribution aspects of the compiler. -//! -//! This module is responsible for creating tarballs of the standard library, -//! compiler, and documentation. This ends up being what we distribute to -//! everyone as well. -//! -//! No tarball is actually created literally in this file, but rather we shell -//! out to `rust-installer` still. This may one day be replaced with bits and -//! pieces of `rustup.rs`! - -use std::collections::HashSet; -use std::ffi::OsStr; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::{env, fs}; - -use object::BinaryFormat; -use object::read::archive::ArchiveFile; - -use crate::core::build_steps::doc::DocumentationFormat; -use crate::core::build_steps::tool::{self, Tool}; -use crate::core::build_steps::vendor::default_paths_to_vendor; -use crate::core::build_steps::{compile, llvm}; -use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::core::config::TargetSelection; -use crate::utils::channel::{self, Info}; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::helpers::{ - exe, is_dylib, move_file, t, target_supports_cranelift_backend, timeit, -}; -use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; -use crate::{Compiler, DependencyType, LLVM_TOOLS, Mode}; - -pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { - format!("{}-{}", component, builder.rust_package_vers()) -} - -pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("dist") -} - -pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("tmp/dist") -} - -fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { - if !builder.config.extended { - return false; - } - builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Docs { - pub host: TargetSelection, -} - -impl Step for Docs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Docs { host: run.target }); - } - - /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let dest = "share/doc/rust/html"; - - let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); - tarball.set_product_name("Rust Documentation"); - tarball.add_bulk_dir(builder.doc_out(host), dest); - tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, 0o644); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct JsonDocs { - pub host: TargetSelection, -} - -impl Step for JsonDocs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs-json").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(JsonDocs { host: run.target }); - } - - /// Builds the `rust-docs-json` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.ensure(crate::core::build_steps::doc::Std::new( - builder.top_stage, - host, - DocumentationFormat::Json, - )); - - let dest = "share/doc/rust/json"; - - let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); - tarball.set_product_name("Rust Documentation In JSON Format"); - tarball.is_preview(true); - tarball.add_bulk_dir(builder.json_doc_out(host), dest); - Some(tarball.generate()) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustcDocs { - pub host: TargetSelection, -} - -impl Step for RustcDocs { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-docs").default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDocs { host: run.target }); - } - - /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); - tarball.set_product_name("Rustc Documentation"); - tarball.add_bulk_dir(builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); - Some(tarball.generate()) - } -} - -fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { - let mut found = Vec::with_capacity(files.len()); - - for file in files { - let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); - - if let Some(file_path) = file_path { - found.push(file_path); - } else { - panic!("Could not find '{file}' in {path:?}"); - } - } - - found -} - -fn make_win_dist( - rust_root: &Path, - plat_root: &Path, - target: TargetSelection, - builder: &Builder<'_>, -) { - if builder.config.dry_run() { - return; - } - - //Ask gcc where it keeps its stuff - let mut cmd = command(builder.cc(target)); - cmd.arg("-print-search-dirs"); - let gcc_out = cmd.run_capture_stdout(builder).stdout(); - - let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); - let mut lib_path = Vec::new(); - - for line in gcc_out.lines() { - let idx = line.find(':').unwrap(); - let key = &line[..idx]; - let trim_chars: &[_] = &[' ', '=']; - let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); - - if key == "programs" { - bin_path.extend(value); - } else if key == "libraries" { - lib_path.extend(value); - } - } - - let compiler = if target == "i686-pc-windows-gnu" { - "i686-w64-mingw32-gcc.exe" - } else if target == "x86_64-pc-windows-gnu" { - "x86_64-w64-mingw32-gcc.exe" - } else { - "gcc.exe" - }; - let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; - let mut rustc_dlls = vec!["libwinpthread-1.dll"]; - if target.starts_with("i686-") { - rustc_dlls.push("libgcc_s_dw2-1.dll"); - } else { - rustc_dlls.push("libgcc_s_seh-1.dll"); - } - - // Libraries necessary to link the windows-gnu toolchains. - // System libraries will be preferred if they are available (see #67429). - let target_libs = [ - //MinGW libs - "libgcc.a", - "libgcc_eh.a", - "libgcc_s.a", - "libm.a", - "libmingw32.a", - "libmingwex.a", - "libstdc++.a", - "libiconv.a", - "libmoldname.a", - "libpthread.a", - //Windows import libs - //This should contain only the set of libraries necessary to link the standard library. - "libadvapi32.a", - "libbcrypt.a", - "libcomctl32.a", - "libcomdlg32.a", - "libcredui.a", - "libcrypt32.a", - "libdbghelp.a", - "libgdi32.a", - "libimagehlp.a", - "libiphlpapi.a", - "libkernel32.a", - "libmsimg32.a", - "libmsvcrt.a", - "libntdll.a", - "libodbc32.a", - "libole32.a", - "liboleaut32.a", - "libopengl32.a", - "libpsapi.a", - "librpcrt4.a", - "libsecur32.a", - "libsetupapi.a", - "libshell32.a", - "libsynchronization.a", - "libuser32.a", - "libuserenv.a", - "libuuid.a", - "libwinhttp.a", - "libwinmm.a", - "libwinspool.a", - "libws2_32.a", - "libwsock32.a", - ]; - - //Find mingw artifacts we want to bundle - let target_tools = find_files(&target_tools, &bin_path); - let rustc_dlls = find_files(&rustc_dlls, &bin_path); - let target_libs = find_files(&target_libs, &lib_path); - - // Copy runtime dlls next to rustc.exe - let rust_bin_dir = rust_root.join("bin/"); - fs::create_dir_all(&rust_bin_dir).expect("creating rust_bin_dir failed"); - for src in &rustc_dlls { - builder.copy_link_to_folder(src, &rust_bin_dir); - } - - if builder.config.lld_enabled { - // rust-lld.exe also needs runtime dlls - let rust_target_bin_dir = rust_root.join("lib/rustlib").join(target).join("bin"); - fs::create_dir_all(&rust_target_bin_dir).expect("creating rust_target_bin_dir failed"); - for src in &rustc_dlls { - builder.copy_link_to_folder(src, &rust_target_bin_dir); - } - } - - //Copy platform tools to platform-specific bin directory - let plat_target_bin_self_contained_dir = - plat_root.join("lib/rustlib").join(target).join("bin/self-contained"); - fs::create_dir_all(&plat_target_bin_self_contained_dir) - .expect("creating plat_target_bin_self_contained_dir failed"); - for src in target_tools { - builder.copy_link_to_folder(&src, &plat_target_bin_self_contained_dir); - } - - // Warn windows-gnu users that the bundled GCC cannot compile C files - builder.create( - &plat_target_bin_self_contained_dir.join("GCC-WARNING.txt"), - "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ - used as a linker. In order to be able to compile projects containing C code use \ - the GCC provided by MinGW or Cygwin.", - ); - - //Copy platform libs to platform-specific lib directory - let plat_target_lib_self_contained_dir = - plat_root.join("lib/rustlib").join(target).join("lib/self-contained"); - fs::create_dir_all(&plat_target_lib_self_contained_dir) - .expect("creating plat_target_lib_self_contained_dir failed"); - for src in target_libs { - builder.copy_link_to_folder(&src, &plat_target_lib_self_contained_dir); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Mingw { - pub host: TargetSelection, -} - -impl Step for Mingw { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-mingw") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Mingw { host: run.target }); - } - - /// Builds the `rust-mingw` installer component. - /// - /// This contains all the bits and pieces to run the MinGW Windows targets - /// without any extra installed software (e.g., we bundle gcc, libraries, etc). - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { - return None; - } - - let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); - tarball.set_product_name("Rust MinGW"); - - // The first argument is a "temporary directory" which is just - // thrown away (this contains the runtime DLLs included in the rustc package - // above) and the second argument is where to place all the MinGW components - // (which is what we want). - make_win_dist(&tmpdir(builder), tarball.image_dir(), host, builder); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub compiler: Compiler, -} - -impl Step for Rustc { - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder - .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); - } - - /// Creates the `rustc` installer component. - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let compiler = self.compiler; - let host = self.compiler.host; - - let tarball = Tarball::new(builder, "rustc", &host.triple); - - // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, tarball.image_dir()); - - // On MinGW we've got a few runtime DLL dependencies that we need to - // include. The first argument to this script is where to put these DLLs - // (the image we're creating), and the second argument is a junk directory - // to ignore all other MinGW stuff the script creates. - // - // On 32-bit MinGW we're always including a DLL which needs some extra - // licenses to distribute. On 64-bit MinGW we don't actually distribute - // anything requiring us to distribute a license, but it's likely the - // install will *also* include the rust-mingw package, which also needs - // licenses, so to be safe we just include it here in all MinGW packages. - if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { - make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); - tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); - } - - return tarball.generate(); - - fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { - let host = compiler.host; - let src = builder.sysroot(compiler); - - // Copy rustc binary - t!(fs::create_dir_all(image.join("bin"))); - builder.cp_link_r(&src.join("bin"), &image.join("bin")); - - // If enabled, copy rustdoc binary - if builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) - { - let rustdoc = builder.rustdoc(compiler); - builder.install(&rustdoc, &image.join("bin"), 0o755); - } - - if let Some(ra_proc_macro_srv) = builder.ensure_if_default( - tool::RustAnalyzerProcMacroSrv { - compiler: builder.compiler_for( - compiler.stage, - builder.config.build, - compiler.host, - ), - target: compiler.host, - }, - builder.kind, - ) { - builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); - } - - let libdir_relative = builder.libdir_relative(compiler); - - // Copy runtime DLLs needed by the compiler - if libdir_relative.to_str() != Some("bin") { - let libdir = builder.rustc_libdir(compiler); - for entry in builder.read_dir(&libdir) { - if is_dylib(&entry.path()) { - // Don't use custom libdir here because ^lib/ will be resolved again - // with installer - builder.install(&entry.path(), &image.join("lib"), 0o644); - } - } - } - - // Copy libLLVM.so to the lib dir as well, if needed. While not - // technically needed by rustc itself it's needed by lots of other - // components like the llvm tools and LLD. LLD is included below and - // tools/LLDB come later, so let's just throw it in the rustc - // component for now. - maybe_install_llvm_runtime(builder, host, image); - - let dst_dir = image.join("lib/rustlib").join(host).join("bin"); - t!(fs::create_dir_all(&dst_dir)); - - // Copy over lld if it's there - if builder.config.lld_enabled { - let src_dir = builder.sysroot_target_bindir(compiler, host); - let rust_lld = exe("rust-lld", compiler.host); - builder.copy_link(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); - let self_contained_lld_src_dir = src_dir.join("gcc-ld"); - let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); - t!(fs::create_dir(&self_contained_lld_dst_dir)); - for name in crate::LLD_FILE_NAMES { - let exe_name = exe(name, compiler.host); - builder.copy_link( - &self_contained_lld_src_dir.join(&exe_name), - &self_contained_lld_dst_dir.join(&exe_name), - ); - } - } - - if builder.config.llvm_enabled(compiler.host) && builder.config.llvm_tools_enabled { - let src_dir = builder.sysroot_target_bindir(compiler, host); - let llvm_objcopy = exe("llvm-objcopy", compiler.host); - let rust_objcopy = exe("rust-objcopy", compiler.host); - builder.copy_link(&src_dir.join(&llvm_objcopy), &dst_dir.join(&rust_objcopy)); - } - - if builder.tool_enabled("wasm-component-ld") { - let src_dir = builder.sysroot_target_bindir(compiler, host); - let ld = exe("wasm-component-ld", compiler.host); - builder.copy_link(&src_dir.join(&ld), &dst_dir.join(&ld)); - } - - // Man pages - t!(fs::create_dir_all(image.join("share/man/man1"))); - let man_src = builder.src.join("src/doc/man"); - let man_dst = image.join("share/man/man1"); - - // don't use our `bootstrap::{copy_internal, cp_r}`, because those try - // to hardlink, and we don't want to edit the source templates - for file_entry in builder.read_dir(&man_src) { - let page_src = file_entry.path(); - let page_dst = man_dst.join(file_entry.file_name()); - let src_text = t!(std::fs::read_to_string(&page_src)); - let new_text = src_text.replace("", &builder.version); - t!(std::fs::write(&page_dst, &new_text)); - t!(fs::copy(&page_src, &page_dst)); - } - - // Debugger scripts - builder.ensure(DebuggerScripts { sysroot: image.to_owned(), host }); - - // Misc license info - let cp = |file: &str| { - builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct DebuggerScripts { - pub sysroot: PathBuf, - pub host: TargetSelection, -} - -impl Step for DebuggerScripts { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Copies debugger scripts for `target` into the `sysroot` specified. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let sysroot = self.sysroot; - let dst = sysroot.join("lib/rustlib/etc"); - t!(fs::create_dir_all(&dst)); - let cp_debugger_script = |file: &str| { - builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); - }; - if host.contains("windows-msvc") { - // windbg debugger scripts - builder.install( - &builder.src.join("src/etc/rust-windbg.cmd"), - &sysroot.join("bin"), - 0o755, - ); - - cp_debugger_script("natvis/intrinsic.natvis"); - cp_debugger_script("natvis/liballoc.natvis"); - cp_debugger_script("natvis/libcore.natvis"); - cp_debugger_script("natvis/libstd.natvis"); - } else { - cp_debugger_script("rust_types.py"); - - // gdb debugger scripts - builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); - builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("gdb_load_rust_pretty_printers.py"); - cp_debugger_script("gdb_lookup.py"); - cp_debugger_script("gdb_providers.py"); - - // lldb debugger scripts - builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("lldb_lookup.py"); - cp_debugger_script("lldb_providers.py"); - cp_debugger_script("lldb_commands") - } - } -} - -fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - true - } else { - false - } -} - -/// Check that all objects in rlibs for UEFI targets are COFF. This -/// ensures that the C compiler isn't producing ELF objects, which would -/// not link correctly with the COFF objects. -fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { - if !target.ends_with("-uefi") { - return; - } - - for (path, _) in builder.read_stamp_file(stamp) { - if path.extension() != Some(OsStr::new("rlib")) { - continue; - } - - let data = t!(fs::read(&path)); - let data = data.as_slice(); - let archive = t!(ArchiveFile::parse(data)); - for member in archive.members() { - let member = t!(member); - let member_data = t!(member.data(data)); - - let is_coff = match object::File::parse(member_data) { - Ok(member_file) => member_file.format() == BinaryFormat::Coff, - Err(_) => false, - }; - - if !is_coff { - let member_name = String::from_utf8_lossy(member.name()); - panic!("member {} in {} is not COFF", member_name, path.display()); - } - } - } -} - -/// Copy stamped files into an image's `target/lib` directory. -fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { - let dst = image.join("lib/rustlib").join(target).join("lib"); - let self_contained_dst = dst.join("self-contained"); - t!(fs::create_dir_all(&dst)); - t!(fs::create_dir_all(&self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - if dependency_type == DependencyType::TargetSelfContained { - builder.copy_link(&path, &self_contained_dst.join(path.file_name().unwrap())); - } else if dependency_type == DependencyType::Target || builder.config.build == target { - builder.copy_link(&path, &dst.join(path.file_name().unwrap())); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Std { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Std { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-std") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Std { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Std::new(compiler, target)); - - let mut tarball = Tarball::new(builder, "rust-std", &target.triple); - tarball.include_target_in_component_name(true); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - verify_uefi_rlib_format(builder, target, &stamp); - copy_target_libs(builder, target, tarball.image_dir(), &stamp); - - Some(tarball.generate()) - } -} - -/// Tarball containing the compiler that gets downloaded and used by -/// `rust.download-rustc`. -/// -/// (Don't confuse this with [`RustDev`], without the `c`!) -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct RustcDev { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustcDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDev { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Rustc::new(compiler, target)); - - let tarball = Tarball::new(builder, "rustc-dev", &target.triple); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, tarball.image_dir(), &stamp); - - let src_files = &["Cargo.lock"]; - // This is the reduced set of paths which will become the rustc-dev component - // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["compiler"], - &[], - &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), - ); - for file in src_files { - tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); - } - - Some(tarball.generate()) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Analysis { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Analysis { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "analysis"); - run.alias("rust-analysis").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Analysis { - // Find the actual compiler (handling the full bootstrap option) which - // produced the save-analysis data because that data isn't copied - // through the sysroot uplifting. - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - /// Creates a tarball of (degenerate) save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if compiler.host != builder.config.build { - return None; - } - - let src = builder - .stage_out(compiler, Mode::Std) - .join(target) - .join(builder.cargo_dir()) - .join("deps") - .join("save-analysis"); - - // Write a file indicating that this component has been removed. - t!(std::fs::create_dir_all(&src)); - let mut removed = src.clone(); - removed.push("removed.json"); - let mut f = t!(std::fs::File::create(removed)); - t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); - - let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); - tarball.include_target_in_component_name(true); - tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); - Some(tarball.generate()) - } -} - -/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to -/// `dst_dir`. -fn copy_src_dirs( - builder: &Builder<'_>, - base: &Path, - src_dirs: &[&str], - exclude_dirs: &[&str], - dst_dir: &Path, -) { - fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { - let spath = match path.to_str() { - Some(path) => path, - None => return false, - }; - if spath.ends_with('~') || spath.ends_with(".pyc") { - return false; - } - - const LLVM_PROJECTS: &[&str] = &[ - "llvm-project/clang", - "llvm-project\\clang", - "llvm-project/libunwind", - "llvm-project\\libunwind", - "llvm-project/lld", - "llvm-project\\lld", - "llvm-project/lldb", - "llvm-project\\lldb", - "llvm-project/llvm", - "llvm-project\\llvm", - "llvm-project/compiler-rt", - "llvm-project\\compiler-rt", - "llvm-project/cmake", - "llvm-project\\cmake", - "llvm-project/runtimes", - "llvm-project\\runtimes", - ]; - if spath.contains("llvm-project") - && !spath.ends_with("llvm-project") - && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) - { - return false; - } - - const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; - if LLVM_TEST.iter().any(|path| spath.contains(path)) - && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) - { - return false; - } - - // Cargo tests use some files like `.gitignore` that we would otherwise exclude. - const CARGO_TESTS: &[&str] = &["tools/cargo/tests", "tools\\cargo\\tests"]; - if CARGO_TESTS.iter().any(|path| spath.contains(path)) { - return true; - } - - let full_path = Path::new(dir).join(path); - if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { - return false; - } - - let excludes = [ - "CVS", - "RCS", - "SCCS", - ".git", - ".gitignore", - ".gitmodules", - ".gitattributes", - ".cvsignore", - ".svn", - ".arch-ids", - "{arch}", - "=RELEASE-ID", - "=meta-update", - "=update", - ".bzr", - ".bzrignore", - ".bzrtags", - ".hg", - ".hgignore", - ".hgrags", - "_darcs", - ]; - !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) - } - - // Copy the directories using our filter - for item in src_dirs { - let dst = &dst_dir.join(item); - t!(fs::create_dir_all(dst)); - builder - .cp_link_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Src; - -impl Step for Src { - /// The output path of the src installer tarball - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-src") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src); - } - - /// Creates the `rust-src` installer component - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - if !builder.config.dry_run() { - builder.require_submodule("src/llvm-project", None); - } - - let tarball = Tarball::new_targetless(builder, "rust-src"); - - // A lot of tools expect the rust-src component to be entirely in this directory, so if you - // change that (e.g. by adding another directory `lib/rustlib/src/foo` or - // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions - // and fix them... - // - // NOTE: if you update the paths here, you also should update the "virtual" path - // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); - - // This is the reduced set of paths which will become the rust-src component - // (essentially libstd and all of its path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["library", "src/llvm-project/libunwind"], - &[ - // not needed and contains symlinks which rustup currently - // chokes on when unpacking. - "library/backtrace/crates", - // these are 30MB combined and aren't necessary for building - // the standard library. - "library/stdarch/Cargo.toml", - "library/stdarch/crates/stdarch-verify", - "library/stdarch/crates/intrinsic-test", - ], - &dst_src, - ); - - tarball.generate() - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct PlainSourceTarball; - -impl Step for PlainSourceTarball { - /// Produces the location of the tarball generated - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-src").default_condition(builder.config.rust_dist_src) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(PlainSourceTarball); - } - - /// Creates the plain source tarball - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which - // means neither rustup nor rustup-toolchain-install-master know how to download it. - // It also contains symbolic links, unlike other any other dist tarball. - // It's used for distros building rustc from source in a pre-vendored environment. - let mut tarball = Tarball::new(builder, "rustc", "src"); - tarball.permit_symlinks(true); - let plain_dst_src = tarball.image_dir(); - - // This is the set of root paths which will become part of the source package - let src_files = [ - "COPYRIGHT", - "LICENSE-APACHE", - "LICENSE-MIT", - "CONTRIBUTING.md", - "README.md", - "RELEASES.md", - "configure", - "x.py", - "config.example.toml", - "Cargo.toml", - "Cargo.lock", - ".gitmodules", - ]; - let src_dirs = ["src", "compiler", "library", "tests"]; - - copy_src_dirs(builder, &builder.src, &src_dirs, &[], plain_dst_src); - - // Copy the files normally - for item in &src_files { - builder.copy_link(&builder.src.join(item), &plain_dst_src.join(item)); - } - - // Create the version file - builder.create(&plain_dst_src.join("version"), &builder.rust_version()); - - // Create the files containing git info, to ensure --version outputs the same. - let write_git_info = |info: Option<&Info>, path: &Path| { - if let Some(info) = info { - t!(std::fs::create_dir_all(path)); - channel::write_commit_hash_file(path, &info.sha); - channel::write_commit_info_file(path, info); - } - }; - write_git_info(builder.rust_info().info(), plain_dst_src); - write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo")); - - if builder.config.dist_vendor { - builder.require_and_update_all_submodules(); - - // Vendor all Cargo dependencies - let mut cmd = command(&builder.initial_cargo); - cmd.arg("vendor").arg("--versioned-dirs"); - - for p in default_paths_to_vendor(builder) { - cmd.arg("--sync").arg(p); - } - - cmd - // Will read the libstd Cargo.toml which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .current_dir(plain_dst_src); - - // Vendor packages that are required by opt-dist to collect PGO profiles. - let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES - .iter() - .chain(build_helper::RUSTC_PGO_CRATES) - .map(|pkg| { - let mut manifest_path = - builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); - manifest_path.push(pkg); - manifest_path.push("Cargo.toml"); - manifest_path - }); - for manifest_path in pkgs_for_pgo_training { - cmd.arg("--sync").arg(manifest_path); - } - - let config = cmd.run_capture(builder).stdout(); - - let cargo_config_dir = plain_dst_src.join(".cargo"); - builder.create_dir(&cargo_config_dir); - builder.create(&cargo_config_dir.join("config.toml"), &config); - } - - // Delete extraneous directories - // FIXME: if we're managed by git, we should probably instead ask git if the given path - // is managed by it? - for entry in walkdir::WalkDir::new(tarball.image_dir()) - .follow_links(true) - .into_iter() - .filter_map(|e| e.ok()) - { - if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__")) - { - t!(fs::remove_dir_all(entry.path())); - } - } - - tarball.bare() - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "cargo"); - run.alias("cargo").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let cargo = builder.ensure(tool::Cargo { compiler, target }); - let src = builder.src.join("src/tools/cargo"); - let etc = src.join("src/etc"); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "cargo", &target.triple); - tarball.set_overlay(OverlayKind::Cargo); - - tarball.add_file(cargo, "bin", 0o755); - tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); - tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); - tarball.add_dir(etc.join("man"), "share/man/man1"); - tarball.add_legal_and_readme_to("share/doc/cargo"); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Rls { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rls { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "rls"); - run.alias("rls").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rls { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rls = builder.ensure(tool::Rls { compiler, target, extra_features: Vec::new() }); - - let mut tarball = Tarball::new(builder, "rls", &target.triple); - tarball.set_overlay(OverlayKind::Rls); - tarball.is_preview(true); - tarball.add_file(rls, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rls"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "rust-analyzer"); - run.alias("rust-analyzer").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target }); - - let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); - tarball.set_overlay(OverlayKind::RustAnalyzer); - tarball.is_preview(true); - tarball.add_file(rust_analyzer, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Clippy { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Clippy { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "clippy"); - run.alias("clippy").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - // Prepare the image directory - // We expect clippy to build, because we've exited this step above if tool - // state for clippy isn't testing. - let clippy = builder.ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }); - let cargoclippy = - builder.ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }); - - let mut tarball = Tarball::new(builder, "clippy", &target.triple); - tarball.set_overlay(OverlayKind::Clippy); - tarball.is_preview(true); - tarball.add_file(clippy, "bin", 0o755); - tarball.add_file(cargoclippy, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/clippy"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Miri { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Miri { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "miri"); - run.alias("miri").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - // This prevents miri from being built for "dist" or "install" - // on the stable/beta channels. It is a nightly-only tool and should - // not be included. - if !builder.build.unstable_features() { - return None; - } - let compiler = self.compiler; - let target = self.target; - - let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() }); - let cargomiri = - builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() }); - - let mut tarball = Tarball::new(builder, "miri", &target.triple); - tarball.set_overlay(OverlayKind::Miri); - tarball.is_preview(true); - tarball.add_file(miri, "bin", 0o755); - tarball.add_file(cargomiri, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/miri"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct CodegenBackend { - pub compiler: Compiler, - pub backend: String, -} - -impl Step for CodegenBackend { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc_codegen_cranelift") - } - - fn make_run(run: RunConfig<'_>) { - for backend in run.builder.config.codegen_backends(run.target) { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - run.builder.ensure(CodegenBackend { - compiler: run.builder.compiler(run.builder.top_stage, run.target), - backend: backend.clone(), - }); - } - } - - fn run(self, builder: &Builder<'_>) -> Option { - if builder.config.dry_run() { - return None; - } - - // This prevents rustc_codegen_cranelift from being built for "dist" - // or "install" on the stable/beta channels. It is not yet stable and - // should not be included. - if !builder.build.unstable_features() { - return None; - } - - if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend.to_string()) - { - return None; - } - - if self.backend == "cranelift" && !target_supports_cranelift_backend(self.compiler.host) { - builder.info("target not supported by rustc_codegen_cranelift. skipping"); - return None; - } - - let compiler = self.compiler; - let backend = self.backend; - - let mut tarball = - Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); - if backend == "cranelift" { - tarball.set_overlay(OverlayKind::RustcCodegenCranelift); - } else { - panic!("Unknown backend rustc_codegen_{}", backend); - } - tarball.is_preview(true); - tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); - - let src = builder.sysroot(compiler); - let backends_src = builder.sysroot_codegen_backends(compiler); - let backends_rel = backends_src - .strip_prefix(src) - .unwrap() - .strip_prefix(builder.sysroot_libdir_relative(compiler)) - .unwrap(); - // Don't use custom libdir here because ^lib/ will be resolved again with installer - let backends_dst = PathBuf::from("lib").join(backends_rel); - - let backend_name = format!("rustc_codegen_{}", backend); - let mut found_backend = false; - for backend in fs::read_dir(&backends_src).unwrap() { - let file_name = backend.unwrap().file_name(); - if file_name.to_str().unwrap().contains(&backend_name) { - tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); - found_backend = true; - } - } - assert!(found_backend); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Rustfmt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rustfmt { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "rustfmt"); - run.alias("rustfmt").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rustfmt = - builder.ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }); - let cargofmt = - builder.ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }); - let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); - tarball.set_overlay(OverlayKind::Rustfmt); - tarball.is_preview(true); - tarball.add_file(rustfmt, "bin", 0o755); - tarball.add_file(cargofmt, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rustfmt"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct Extended { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Step for Extended { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("extended").default_condition(builder.config.extended) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Extended { - stage: run.builder.top_stage, - host: run.builder.config.build, - target: run.target, - }); - } - - /// Creates a combined installer for the specified target in the provided stage. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let stage = self.stage; - let compiler = builder.compiler_for(self.stage, self.host, self.target); - - builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); - - let mut tarballs = Vec::new(); - let mut built_tools = HashSet::new(); - macro_rules! add_component { - ($name:expr => $step:expr) => { - if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { - tarballs.push(tarball); - built_tools.insert($name); - } - }; - } - - // When rust-std package split from rustc, we needed to ensure that during - // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering - // the std files during uninstall. To do this ensure that rustc comes - // before rust-std in the list below. - tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); - tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); - - if target.is_windows_gnu() { - tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); - } - - add_component!("rust-docs" => Docs { host: target }); - add_component!("rust-json-docs" => JsonDocs { host: target }); - add_component!("cargo" => Cargo { compiler, target }); - add_component!("rustfmt" => Rustfmt { compiler, target }); - add_component!("rls" => Rls { compiler, target }); - add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); - add_component!("llvm-components" => LlvmTools { target }); - add_component!("clippy" => Clippy { compiler, target }); - add_component!("miri" => Miri { compiler, target }); - add_component!("analysis" => Analysis { compiler, target }); - add_component!("rustc-codegen-cranelift" => CodegenBackend { - compiler: builder.compiler(stage, target), - backend: "cranelift".to_string(), - }); - add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker {compiler, target}); - - let etc = builder.src.join("src/etc/installer"); - - // Avoid producing tarballs during a dry run. - if builder.config.dry_run() { - return; - } - - let tarball = Tarball::new(builder, "rust", &target.triple); - let generated = tarball.combine(&tarballs); - - let tmp = tmpdir(builder).join("combined-tarball"); - let work = generated.work_dir(); - - let mut license = String::new(); - license += &builder.read(&builder.src.join("COPYRIGHT")); - license += &builder.read(&builder.src.join("LICENSE-APACHE")); - license += &builder.read(&builder.src.join("LICENSE-MIT")); - license.push('\n'); - license.push('\n'); - - let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; - let mut rtf = rtf.to_string(); - rtf.push('\n'); - for line in license.lines() { - rtf.push_str(line); - rtf.push_str("\\line "); - } - rtf.push('}'); - - fn filter(contents: &str, marker: &str) -> String { - let start = format!("tool-{marker}-start"); - let end = format!("tool-{marker}-end"); - let mut lines = Vec::new(); - let mut omitted = false; - for line in contents.lines() { - if line.contains(&start) { - omitted = true; - } else if line.contains(&end) { - omitted = false; - } else if !omitted { - lines.push(line); - } - } - - lines.join("\n") - } - - let xform = |p: &Path| { - let mut contents = t!(fs::read_to_string(p)); - for tool in &["miri", "rust-docs"] { - if !built_tools.contains(tool) { - contents = filter(&contents, tool); - } - } - let ret = tmp.join(p.file_name().unwrap()); - t!(fs::write(&ret, &contents)); - ret - }; - - if target.contains("apple-darwin") { - builder.info("building pkg installer"); - let pkg = tmp.join("pkg"); - let _ = fs::remove_dir_all(&pkg); - - let pkgbuild = |component: &str| { - let mut cmd = command("pkgbuild"); - cmd.arg("--identifier") - .arg(format!("org.rust-lang.{}", component)) - .arg("--scripts") - .arg(pkg.join(component)) - .arg("--nopayload") - .arg(pkg.join(component).with_extension("pkg")); - cmd.run(builder); - }; - - let prepare = |name: &str| { - builder.create_dir(&pkg.join(name)); - builder.cp_link_r( - &work.join(format!("{}-{}", pkgname(builder, name), target.triple)), - &pkg.join(name), - ); - builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); - pkgbuild(name); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-std"); - prepare("rust-analysis"); - prepare("clippy"); - prepare("rust-analyzer"); - for tool in &["rust-docs", "miri", "rustc-codegen-cranelift"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - // create an 'uninstall' package - builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); - pkgbuild("uninstall"); - - builder.create_dir(&pkg.join("res")); - builder.create(&pkg.join("res/LICENSE.txt"), &license); - builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); - let mut cmd = command("productbuild"); - cmd.arg("--distribution") - .arg(xform(&etc.join("pkg/Distribution.xml"))) - .arg("--resources") - .arg(pkg.join("res")) - .arg(distdir(builder).join(format!( - "{}-{}.pkg", - pkgname(builder, "rust"), - target.triple - ))) - .arg("--package-path") - .arg(&pkg); - let _time = timeit(builder); - cmd.run(builder); - } - - if target.is_windows() { - let exe = tmp.join("exe"); - let _ = fs::remove_dir_all(&exe); - - let prepare = |name: &str| { - builder.create_dir(&exe.join(name)); - let dir = if name == "rust-std" || name == "rust-analysis" { - format!("{}-{}", name, target.triple) - } else if name == "rust-analyzer" { - "rust-analyzer-preview".to_string() - } else if name == "clippy" { - "clippy-preview".to_string() - } else if name == "miri" { - "miri-preview".to_string() - } else if name == "rustc-codegen-cranelift" { - // FIXME add installer support for cg_clif once it is ready to be distributed on - // windows. - unreachable!("cg_clif shouldn't be built for windows"); - } else { - name.to_string() - }; - builder.cp_link_r( - &work.join(format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), - &exe.join(name), - ); - builder.remove(&exe.join(name).join("manifest.in")); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-analysis"); - prepare("rust-std"); - for tool in &["clippy", "rust-analyzer", "rust-docs", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - if target.is_windows_gnu() { - prepare("rust-mingw"); - } - - builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); - - // Generate msi installer - let wix_path = env::var_os("WIX") - .expect("`WIX` environment variable must be set for generating MSI installer(s)."); - let wix = PathBuf::from(wix_path); - let heat = wix.join("bin/heat.exe"); - let candle = wix.join("bin/candle.exe"); - let light = wix.join("bin/light.exe"); - - let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rustc") - .args(heat_flags) - .arg("-cg") - .arg("RustcGroup") - .arg("-dr") - .arg("Rustc") - .arg("-var") - .arg("var.RustcDir") - .arg("-out") - .arg(exe.join("RustcGroup.wxs")) - .run(builder); - if built_tools.contains("rust-docs") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")) - .run(builder); - } - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("cargo") - .args(heat_flags) - .arg("-cg") - .arg("CargoGroup") - .arg("-dr") - .arg("Cargo") - .arg("-var") - .arg("var.CargoDir") - .arg("-out") - .arg(exe.join("CargoGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-std") - .args(heat_flags) - .arg("-cg") - .arg("StdGroup") - .arg("-dr") - .arg("Std") - .arg("-var") - .arg("var.StdDir") - .arg("-out") - .arg(exe.join("StdGroup.wxs")) - .run(builder); - if built_tools.contains("rust-analyzer") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analyzer") - .args(heat_flags) - .arg("-cg") - .arg("RustAnalyzerGroup") - .arg("-dr") - .arg("RustAnalyzer") - .arg("-var") - .arg("var.RustAnalyzerDir") - .arg("-out") - .arg(exe.join("RustAnalyzerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - if built_tools.contains("clippy") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("clippy") - .args(heat_flags) - .arg("-cg") - .arg("ClippyGroup") - .arg("-dr") - .arg("Clippy") - .arg("-var") - .arg("var.ClippyDir") - .arg("-out") - .arg(exe.join("ClippyGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - if built_tools.contains("miri") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("miri") - .args(heat_flags) - .arg("-cg") - .arg("MiriGroup") - .arg("-dr") - .arg("Miri") - .arg("-var") - .arg("var.MiriDir") - .arg("-out") - .arg(exe.join("MiriGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analysis") - .args(heat_flags) - .arg("-cg") - .arg("AnalysisGroup") - .arg("-dr") - .arg("Analysis") - .arg("-var") - .arg("var.AnalysisDir") - .arg("-out") - .arg(exe.join("AnalysisGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - if target.is_windows_gnu() { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-mingw") - .args(heat_flags) - .arg("-cg") - .arg("GccGroup") - .arg("-dr") - .arg("Gcc") - .arg("-var") - .arg("var.GccDir") - .arg("-out") - .arg(exe.join("GccGroup.wxs")) - .run(builder); - } - - let candle = |input: &Path| { - let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); - let arch = if target.contains("x86_64") { "x64" } else { "x86" }; - let mut cmd = command(&candle); - cmd.current_dir(&exe) - .arg("-nologo") - .arg("-dRustcDir=rustc") - .arg("-dCargoDir=cargo") - .arg("-dStdDir=rust-std") - .arg("-dAnalysisDir=rust-analysis") - .arg("-arch") - .arg(arch) - .arg("-out") - .arg(&output) - .arg(input); - add_env(builder, &mut cmd, target); - - if built_tools.contains("clippy") { - cmd.arg("-dClippyDir=clippy"); - } - if built_tools.contains("rust-docs") { - cmd.arg("-dDocsDir=rust-docs"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("-dRustAnalyzerDir=rust-analyzer"); - } - if built_tools.contains("miri") { - cmd.arg("-dMiriDir=miri"); - } - if target.is_windows_gnu() { - cmd.arg("-dGccDir=rust-mingw"); - } - cmd.run(builder); - }; - candle(&xform(&etc.join("msi/rust.wxs"))); - candle(&etc.join("msi/ui.wxs")); - candle(&etc.join("msi/rustwelcomedlg.wxs")); - candle("RustcGroup.wxs".as_ref()); - if built_tools.contains("rust-docs") { - candle("DocsGroup.wxs".as_ref()); - } - candle("CargoGroup.wxs".as_ref()); - candle("StdGroup.wxs".as_ref()); - if built_tools.contains("clippy") { - candle("ClippyGroup.wxs".as_ref()); - } - if built_tools.contains("miri") { - candle("MiriGroup.wxs".as_ref()); - } - if built_tools.contains("rust-analyzer") { - candle("RustAnalyzerGroup.wxs".as_ref()); - } - candle("AnalysisGroup.wxs".as_ref()); - - if target.is_windows_gnu() { - candle("GccGroup.wxs".as_ref()); - } - - builder.create(&exe.join("LICENSE.rtf"), &rtf); - builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); - builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); - - builder.info(&format!("building `msi` installer with {light:?}")); - let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); - let mut cmd = command(&light); - cmd.arg("-nologo") - .arg("-ext") - .arg("WixUIExtension") - .arg("-ext") - .arg("WixUtilExtension") - .arg("-out") - .arg(exe.join(&filename)) - .arg("rust.wixobj") - .arg("ui.wixobj") - .arg("rustwelcomedlg.wixobj") - .arg("RustcGroup.wixobj") - .arg("CargoGroup.wixobj") - .arg("StdGroup.wixobj") - .arg("AnalysisGroup.wixobj") - .current_dir(&exe); - - if built_tools.contains("clippy") { - cmd.arg("ClippyGroup.wixobj"); - } - if built_tools.contains("miri") { - cmd.arg("MiriGroup.wixobj"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("RustAnalyzerGroup.wixobj"); - } - if built_tools.contains("rust-docs") { - cmd.arg("DocsGroup.wixobj"); - } - - if target.is_windows_gnu() { - cmd.arg("GccGroup.wixobj"); - } - // ICE57 wrongly complains about the shortcuts - cmd.arg("-sice:ICE57"); - - let _time = timeit(builder); - cmd.run(builder); - - if !builder.config.dry_run() { - t!(move_file(exe.join(&filename), distdir(builder).join(&filename))); - } - } - } -} - -fn add_env(builder: &Builder<'_>, cmd: &mut BootstrapCommand, target: TargetSelection) { - let mut parts = builder.version.split('.'); - cmd.env("CFG_RELEASE_INFO", builder.rust_version()) - .env("CFG_RELEASE_NUM", &builder.version) - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_VER_MAJOR", parts.next().unwrap()) - .env("CFG_VER_MINOR", parts.next().unwrap()) - .env("CFG_VER_PATCH", parts.next().unwrap()) - .env("CFG_VER_BUILD", "0") // just needed to build - .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) - .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) - .env("CFG_BUILD", target.triple) - .env("CFG_CHANNEL", &builder.config.channel); - - if target.contains("windows-gnullvm") { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); - } else if target.is_windows_gnu() { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); - } else { - cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); - } -} - -fn install_llvm_file( - builder: &Builder<'_>, - source: &Path, - destination: &Path, - install_symlink: bool, -) { - if builder.config.dry_run() { - return; - } - - if source.is_symlink() { - // If we have a symlink like libLLVM-18.so -> libLLVM.so.18.1, install the target of the - // symlink, which is what will actually get loaded at runtime. - builder.install(&t!(fs::canonicalize(source)), destination, 0o644); - - let full_dest = destination.join(source.file_name().unwrap()); - if install_symlink { - // For download-ci-llvm, also install the symlink, to match what LLVM does. Using a - // symlink is fine here, as this is not a rustup component. - builder.copy_link(source, &full_dest); - } else { - // Otherwise, replace the symlink with an equivalent linker script. This is used when - // projects like miri link against librustc_driver.so. We don't use a symlink, as - // these are not allowed inside rustup components. - let link = t!(fs::read_link(source)); - let mut linker_script = t!(fs::File::create(full_dest)); - t!(write!(linker_script, "INPUT({})\n", link.display())); - - // We also want the linker script to have the same mtime as the source, otherwise it - // can trigger rebuilds. - let meta = t!(fs::metadata(source)); - if let Ok(mtime) = meta.modified() { - t!(linker_script.set_modified(mtime)); - } - } - } else { - builder.install(source, destination, 0o644); - } -} - -/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. -/// -/// Returns whether the files were actually copied. -fn maybe_install_llvm( - builder: &Builder<'_>, - target: TargetSelection, - dst_libdir: &Path, - install_symlink: bool, -) -> bool { - // If the LLVM was externally provided, then we don't currently copy - // artifacts into the sysroot. This is not necessarily the right - // choice (in particular, it will require the LLVM dylib to be in - // the linker's load path at runtime), but the common use case for - // external LLVMs is distribution provided LLVMs, and in that case - // they're usually in the standard search path (e.g., /usr/lib) and - // copying them here is going to cause problems as we may end up - // with the wrong files and isn't what distributions want. - // - // This behavior may be revisited in the future though. - // - // NOTE: this intentionally doesn't use `is_rust_llvm`; whether this is patched or not doesn't matter, - // we only care if the shared object itself is managed by bootstrap. - // - // If the LLVM is coming from ourselves (just from CI) though, we - // still want to install it, as it otherwise won't be available. - if builder.is_system_llvm(target) { - return false; - } - - // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib - // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely - // clear why this is the case, though. llvm-config will emit the versioned - // paths and we don't want those in the sysroot (as we're expecting - // unversioned paths). - if target.contains("apple-darwin") && builder.llvm_link_shared() { - let src_libdir = builder.llvm_out(target).join("lib"); - let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); - if llvm_dylib_path.exists() { - builder.install(&llvm_dylib_path, dst_libdir, 0o644); - } - !builder.config.dry_run() - } else if let llvm::LlvmBuildStatus::AlreadyBuilt(llvm::LlvmResult { llvm_config, .. }) = - llvm::prebuilt_llvm_config(builder, target, true) - { - let mut cmd = command(llvm_config); - cmd.arg("--libfiles"); - builder.verbose(|| println!("running {cmd:?}")); - let files = cmd.run_capture_stdout(builder).stdout(); - let build_llvm_out = &builder.llvm_out(builder.config.build); - let target_llvm_out = &builder.llvm_out(target); - for file in files.trim_end().split(' ') { - // If we're not using a custom LLVM, make sure we package for the target. - let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { - target_llvm_out.join(relative_path) - } else { - PathBuf::from(file) - }; - install_llvm_file(builder, &file, dst_libdir, install_symlink); - } - !builder.config.dry_run() - } else { - false - } -} - -/// Maybe add libLLVM.so to the target lib-dir for linking. -pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = sysroot.join("lib/rustlib").join(target).join("lib"); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir, false); - } -} - -/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. -pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = - sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir, false); - } -} - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct LlvmTools { - pub target: TargetSelection, -} - -impl Step for LlvmTools { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "llvm-tools"); - - let mut run = run.alias("llvm-tools"); - for tool in LLVM_TOOLS { - run = run.alias(tool); - } - - run.default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LlvmTools { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - fn tools_to_install(paths: &[PathBuf]) -> Vec<&'static str> { - let mut tools = vec![]; - - for path in paths { - let path = path.to_str().unwrap(); - - // Include all tools if path is 'llvm-tools'. - if path == "llvm-tools" { - return LLVM_TOOLS.to_owned(); - } - - for tool in LLVM_TOOLS { - if path == *tool { - tools.push(*tool); - } - } - } - - // If no specific tool is requested, include all tools. - if tools.is_empty() { - tools = LLVM_TOOLS.to_owned(); - } - - tools - } - - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); - return None; - } - } - - builder.ensure(crate::core::build_steps::llvm::Llvm { target }); - - let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); - tarball.set_overlay(OverlayKind::Llvm); - tarball.is_preview(true); - - if builder.config.llvm_tools_enabled { - // Prepare the image directory - let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); - for tool in tools_to_install(&builder.paths) { - let exe = src_bindir.join(exe(tool, target)); - tarball.add_file(&exe, &dst_bindir, 0o755); - } - } - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - maybe_install_llvm_target(builder, target, tarball.image_dir()); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct LlvmBitcodeLinker { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LlvmBitcodeLinker { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(run.builder, "llvm-bitcode-linker"); - run.alias("llvm-bitcode-linker").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LlvmBitcodeLinker { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let llbc_linker = - builder.ensure(tool::LlvmBitcodeLinker { compiler, target, extra_features: vec![] }); - - let self_contained_bin_dir = format!("lib/rustlib/{}/bin/self-contained", target.triple); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "llvm-bitcode-linker", &target.triple); - tarball.set_overlay(OverlayKind::LlvmBitcodeLinker); - tarball.is_preview(true); - - tarball.add_file(llbc_linker, self_contained_bin_dir, 0o755); - - Some(tarball.generate()) - } -} - -/// Tarball intended for internal consumption to ease rustc/std development. -/// -/// Should not be considered stable by end users. -/// -/// In practice, this is the tarball that gets downloaded and used by -/// `llvm.download-ci-llvm`. -/// -/// (Don't confuse this with [`RustcDev`], with a `c`!) -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct RustDev { - pub target: TargetSelection, -} - -impl Step for RustDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDev { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping RustDev ({target}): external LLVM")); - return None; - } - } - - let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); - tarball.set_overlay(OverlayKind::Llvm); - // LLVM requires a shared object symlink to exist on some platforms. - tarball.permit_symlinks(true); - - builder.ensure(crate::core::build_steps::llvm::Llvm { target }); - - let src_bindir = builder.llvm_out(target).join("bin"); - // If updating this, you likely want to change - // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users - // will not pick up the extra file until LLVM gets bumped. - // We should include all the build artifacts obtained from a source build, - // so that you can use the downloadable LLVM as if you’ve just run a full source build. - if src_bindir.exists() { - for entry in walkdir::WalkDir::new(&src_bindir) { - let entry = t!(entry); - if entry.file_type().is_file() && !entry.path_is_symlink() { - let name = entry.file_name().to_str().unwrap(); - tarball.add_file(src_bindir.join(name), "bin", 0o755); - } - } - } - - if builder.config.lld_enabled { - // We want to package `lld` to use it with `download-ci-llvm`. - let lld_out = builder.ensure(crate::core::build_steps::llvm::Lld { target }); - - // We don't build LLD on some platforms, so only add it if it exists - let lld_path = lld_out.join("bin").join(exe("lld", target)); - if lld_path.exists() { - tarball.add_file(lld_path, "bin", 0o755); - } - } - - tarball.add_file(builder.llvm_filecheck(target), "bin", 0o755); - - // Copy the include directory as well; needed mostly to build - // librustc_llvm properly (e.g., llvm-config.h is in here). But also - // just broadly useful to be able to link against the bundled LLVM. - tarball.add_dir(builder.llvm_out(target).join("include"), "include"); - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - let dst_libdir = tarball.image_dir().join("lib"); - maybe_install_llvm(builder, target, &dst_libdir, true); - let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; - t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); - - // Copy the `compiler-rt` source, so that `library/profiler_builtins` - // can potentially use it to build the profiler runtime without needing - // to check out the LLVM submodule. - copy_src_dirs( - builder, - &builder.src.join("src").join("llvm-project"), - &["compiler-rt"], - // The test subdirectory is much larger than the rest of the source, - // and we currently don't use these test files anyway. - &["compiler-rt/test"], - tarball.image_dir(), - ); - - Some(tarball.generate()) - } -} - -/// Tarball intended for internal consumption to ease rustc/std development. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct Bootstrap { - pub target: TargetSelection, -} - -impl Step for Bootstrap { - type Output = Option; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - let tarball = Tarball::new(builder, "bootstrap", &target.triple); - - let bootstrap_outdir = &builder.bootstrap_out; - for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { - tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); - } - - Some(tarball.generate()) - } -} - -/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the -/// release process to avoid cloning the monorepo and building stuff. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct BuildManifest { - pub target: TargetSelection, -} - -impl Step for BuildManifest { - type Output = GeneratedTarball; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let build_manifest = builder.tool_exe(Tool::BuildManifest); - - let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); - tarball.add_file(build_manifest, "bin", 0o755); - tarball.generate() - } -} - -/// Tarball containing artifacts necessary to reproduce the build of rustc. -/// -/// Currently this is the PGO profile data. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct ReproducibleArtifacts { - pub target: TargetSelection, -} - -impl Step for ReproducibleArtifacts { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("reproducible-artifacts") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReproducibleArtifacts { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut added_anything = false; - let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); - if let Some(path) = builder.config.rust_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - for profile in &builder.config.reproducible_artifacts { - tarball.add_file(profile, ".", 0o644); - added_anything = true; - } - if added_anything { Some(tarball.generate()) } else { None } - } -} diff --git a/standalonex/src/src/core/build_steps/doc.rs b/standalonex/src/src/core/build_steps/doc.rs deleted file mode 100644 index 8a9321f8..00000000 --- a/standalonex/src/src/core/build_steps/doc.rs +++ /dev/null @@ -1,1298 +0,0 @@ -//! Documentation generation for bootstrap. -//! -//! This module implements generation for all bits and pieces of documentation -//! for the Rust project. This notably includes suites like the rust book, the -//! nomicon, rust by example, standalone documentation, etc. -//! -//! Everything here is basically just a shim around calling either `rustbook` or -//! `rustdoc`. - -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; -use std::{env, fs, mem}; - -use crate::Mode; -use crate::core::build_steps::compile; -use crate::core::build_steps::tool::{self, SourceType, Tool, prepare_tool_cargo}; -use crate::core::builder::{ - self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, crate_description, -}; -use crate::core::config::{Config, TargetSelection}; -use crate::utils::helpers::{symlink_dir, t, up_to_date}; - -macro_rules! submodule_helper { - ($path:expr, submodule) => { - $path - }; - ($path:expr, submodule = $submodule:literal) => { - $submodule - }; -} - -macro_rules! book { - ($($name:ident, $path:expr, $book_name:expr, $lang:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) { - $( - let path = submodule_helper!( $path, submodule $( = $submodule )? ); - builder.require_submodule(path, None); - )? - builder.ensure(RustbookSrc { - target: self.target, - name: $book_name.to_owned(), - src: builder.src.join($path), - parent: Some(self), - languages: $lang.into(), - rustdoc_compiler: None, - }) - } - } - )+ - } -} - -// NOTE: When adding a book here, make sure to ALSO build the book by -// adding a build step in `src/bootstrap/builder.rs`! -// NOTE: Make sure to add the corresponding submodule when adding a new book. -// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules -// and checking against it?). -book!( - CargoBook, "src/tools/cargo/src/doc", "cargo", &[], submodule = "src/tools/cargo"; - ClippyBook, "src/tools/clippy/book", "clippy", &[]; - EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule; - Nomicon, "src/doc/nomicon", "nomicon", &[], submodule; - RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja", "zh"], submodule; - RustdocBook, "src/doc/rustdoc", "rustdoc", &[]; - StyleGuide, "src/doc/style-guide", "style-guide", &[]; -); - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBook { - target: TargetSelection, -} - -impl Step for UnstableBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/unstable-book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBook { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(UnstableBookGen { target: self.target }); - builder.ensure(RustbookSrc { - target: self.target, - name: "unstable-book".to_owned(), - src: builder.md_doc_out(self.target).join("unstable-book"), - parent: Some(self), - languages: vec![], - rustdoc_compiler: None, - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct RustbookSrc { - target: TargetSelection, - name: String, - src: PathBuf, - parent: Option

, - languages: Vec<&'static str>, - rustdoc_compiler: Option, -} - -impl Step for RustbookSrc

{ - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. - /// - /// This will not actually generate any documentation if the documentation has - /// already been generated. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let name = self.name; - let src = self.src; - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let out = out.join(&name); - let index = out.join("index.html"); - let rustbook = builder.tool_exe(Tool::Rustbook); - - if !builder.config.dry_run() - && (!up_to_date(&src, &index) || !up_to_date(&rustbook, &index)) - { - builder.info(&format!("Rustbook ({target}) - {name}")); - let _ = fs::remove_dir_all(&out); - - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - - if let Some(compiler) = self.rustdoc_compiler { - let mut rustdoc = builder.rustdoc(compiler); - rustdoc.pop(); - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = - env::join_paths(std::iter::once(rustdoc).chain(env::split_paths(&old_path))) - .expect("could not add rustdoc to PATH"); - - rustbook_cmd.env("PATH", new_path); - builder.add_rustc_lib_path(compiler, &mut rustbook_cmd); - } - - rustbook_cmd - .arg("build") - .arg(&src) - .arg("-d") - .arg(&out) - .arg("--rust-root") - .arg(&builder.src) - .run(builder); - - for lang in &self.languages { - let out = out.join(lang); - - builder.info(&format!("Rustbook ({target}) - {name} - {lang}")); - let _ = fs::remove_dir_all(&out); - - builder - .tool_cmd(Tool::Rustbook) - .arg("build") - .arg(&src) - .arg("-d") - .arg(&out) - .arg("-l") - .arg(lang) - .run(builder); - } - } - - if self.parent.is_some() { - builder.maybe_open_in_browser::

(index) - } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct TheBook { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for TheBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TheBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Builds the book and associated stuff. - /// - /// We need to build: - /// - /// * Book - /// * Older edition redirects - /// * Version info and CSS - /// * Index page - /// * Redirect pages - fn run(self, builder: &Builder<'_>) { - builder.require_submodule("src/doc/book", None); - - let compiler = self.compiler; - let target = self.target; - - let absolute_path = builder.src.join("src/doc/book"); - let redirect_path = absolute_path.join("redirects"); - - // build book - builder.ensure(RustbookSrc { - target, - name: "book".to_owned(), - src: absolute_path.clone(), - parent: Some(self), - languages: vec![], - rustdoc_compiler: None, - }); - - // building older edition redirects - for edition in &["first-edition", "second-edition", "2018-edition"] { - builder.ensure(RustbookSrc { - target, - name: format!("book/{edition}"), - src: absolute_path.join(edition), - // There should only be one book that is marked as the parent for each target, so - // treat the other editions as not having a parent. - parent: Option::::None, - languages: vec![], - rustdoc_compiler: None, - }); - } - - // build the version info page and CSS - let shared_assets = builder.ensure(SharedAssets { target }); - - // build the redirect pages - let _guard = builder.msg_doc(compiler, "book redirect pages", target); - for file in t!(fs::read_dir(redirect_path)) { - let file = t!(file); - let path = file.path(); - let path = path.to_str().unwrap(); - - invoke_rustdoc(builder, compiler, &shared_assets, target, path); - } - } -} - -fn invoke_rustdoc( - builder: &Builder<'_>, - compiler: Compiler, - shared_assets: &SharedAssetsPaths, - target: TargetSelection, - markdown: &str, -) { - let out = builder.doc_out(target); - - let path = builder.src.join("src/doc").join(markdown); - - let header = builder.src.join("src/doc/redirect.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - - let mut cmd = builder.rustdoc_cmd(compiler); - - let out = out.join("book"); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&shared_assets.version_info) - .arg("--html-in-header") - .arg(&header) - .arg("--markdown-no-toc") - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path) - .arg("--markdown-css") - .arg("../rust.css") - .arg("-Zunstable-options"); - - if !builder.config.docs_minification { - cmd.arg("--disable-minification"); - } - - cmd.run(builder); -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Standalone { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for Standalone { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc").alias("standalone").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Standalone { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Generates all standalone documentation as compiled by the rustdoc in `stage` - /// for the `target` into `out`. - /// - /// This will list all of `src/doc` looking for markdown files and appropriately - /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and - /// `STAMP` along with providing the various header/footer HTML we've customized. - /// - /// In the end, this is just a glorified wrapper around rustdoc! - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - let _guard = builder.msg_doc(compiler, "standalone", target); - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; - - let favicon = builder.src.join("src/doc/favicon.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - let full_toc = builder.src.join("src/doc/full-toc.inc"); - - for file in t!(fs::read_dir(builder.src.join("src/doc"))) { - let file = t!(file); - let path = file.path(); - let filename = path.file_name().unwrap().to_str().unwrap(); - if !filename.ends_with(".md") || filename == "README.md" { - continue; - } - - let html = out.join(filename).with_extension("html"); - let rustdoc = builder.rustdoc(compiler); - if up_to_date(&path, &html) - && up_to_date(&footer, &html) - && up_to_date(&favicon, &html) - && up_to_date(&full_toc, &html) - && (builder.config.dry_run() || up_to_date(&version_info, &html)) - && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) - { - continue; - } - - let mut cmd = builder.rustdoc_cmd(compiler); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&version_info) - .arg("--html-in-header") - .arg(&favicon) - .arg("--markdown-no-toc") - .arg("-Zunstable-options") - .arg("--index-page") - .arg(builder.src.join("src/doc/index.md")) - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path); - - if !builder.config.docs_minification { - cmd.arg("--disable-minification"); - } - - if filename == "not_found.md" { - cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css"); - } else { - cmd.arg("--markdown-css").arg("rust.css"); - } - cmd.run(builder); - } - - // We open doc/index.html as the default if invoked as `x.py doc --open` - // with no particular explicit doc requested (e.g. library/core). - if builder.paths.is_empty() || builder.was_invoked_explicitly::(Kind::Doc) { - let index = out.join("index.html"); - builder.open_in_browser(index); - } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Releases { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for Releases { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("RELEASES.md").alias("releases").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Releases { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Generates HTML release notes to include in the final docs bundle. - /// - /// This uses the same stylesheet and other tools as Standalone, but the - /// RELEASES.md file is included at the root of the repository and gets - /// the headline added. In the end, the conversion is done by Rustdoc. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - let _guard = builder.msg_doc(compiler, "releases", target); - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - builder.ensure(Standalone { - compiler: builder.compiler(builder.top_stage, builder.config.build), - target, - }); - - let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; - - let favicon = builder.src.join("src/doc/favicon.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - let full_toc = builder.src.join("src/doc/full-toc.inc"); - - let html = out.join("releases.html"); - let tmppath = out.join("releases.md"); - let inpath = builder.src.join("RELEASES.md"); - let rustdoc = builder.rustdoc(compiler); - if !up_to_date(&inpath, &html) - || !up_to_date(&footer, &html) - || !up_to_date(&favicon, &html) - || !up_to_date(&full_toc, &html) - || !(builder.config.dry_run() - || up_to_date(&version_info, &html) - || up_to_date(&rustdoc, &html)) - { - let mut tmpfile = t!(fs::File::create(&tmppath)); - t!(tmpfile.write_all(b"% Rust Release Notes\n\n")); - t!(io::copy(&mut t!(fs::File::open(&inpath)), &mut tmpfile)); - mem::drop(tmpfile); - let mut cmd = builder.rustdoc_cmd(compiler); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&version_info) - .arg("--html-in-header") - .arg(&favicon) - .arg("--markdown-no-toc") - .arg("--markdown-css") - .arg("rust.css") - .arg("-Zunstable-options") - .arg("--index-page") - .arg(builder.src.join("src/doc/index.md")) - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&tmppath); - - if !builder.config.docs_minification { - cmd.arg("--disable-minification"); - } - - cmd.run(builder); - } - - // We open doc/RELEASES.html as the default if invoked as `x.py doc --open RELEASES.md` - // with no particular explicit doc requested (e.g. library/core). - if builder.was_invoked_explicitly::(Kind::Doc) { - builder.open_in_browser(&html); - } - } -} - -#[derive(Debug, Clone)] -pub struct SharedAssetsPaths { - pub version_info: PathBuf, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct SharedAssets { - target: TargetSelection, -} - -impl Step for SharedAssets { - type Output = SharedAssetsPaths; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Other tasks depend on this, no need to execute it on its own - run.never() - } - - /// Generate shared resources used by other pieces of documentation. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let out = builder.doc_out(self.target); - - let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); - let version_info = out.join("version_info.html"); - if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { - let info = t!(fs::read_to_string(&version_input)) - .replace("VERSION", &builder.rust_release()) - .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) - .replace("STAMP", builder.rust_info().sha().unwrap_or("")); - t!(fs::write(&version_info, info)); - } - - builder.copy_link( - &builder.src.join("src").join("doc").join("rust.css"), - &out.join("rust.css"), - ); - - SharedAssetsPaths { version_info } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct Std { - pub stage: u32, - pub target: TargetSelection, - pub format: DocumentationFormat, - crates: Vec, -} - -impl Std { - pub(crate) fn new(stage: u32, target: TargetSelection, format: DocumentationFormat) -> Self { - Std { stage, target, format, crates: vec![] } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - let crates = compile::std_crates_for_run_make(&run); - run.builder.ensure(Std { - stage: run.builder.top_stage, - target: run.target, - format: if run.builder.config.cmd.json() { - DocumentationFormat::Json - } else { - DocumentationFormat::Html - }, - crates, - }); - } - - /// Compile all standard library documentation. - /// - /// This will generate all documentation for the standard library and its - /// dependencies. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - let crates = if self.crates.is_empty() { - builder - .in_tree_crates("sysroot", Some(target)) - .iter() - .map(|c| c.name.to_string()) - .collect() - } else { - self.crates - }; - - let out = match self.format { - DocumentationFormat::Html => builder.doc_out(target), - DocumentationFormat::Json => builder.json_doc_out(target), - }; - - t!(fs::create_dir_all(&out)); - - if self.format == DocumentationFormat::Html { - builder.ensure(SharedAssets { target: self.target }); - } - - let index_page = builder - .src - .join("src/doc/index.md") - .into_os_string() - .into_string() - .expect("non-utf8 paths are unsupported"); - let mut extra_args = match self.format { - DocumentationFormat::Html => { - vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page] - } - DocumentationFormat::Json => vec!["--output-format", "json"], - }; - - if !builder.config.docs_minification { - extra_args.push("--disable-minification"); - } - // For `--index-page` and `--output-format=json`. - extra_args.push("-Zunstable-options"); - - doc_std(builder, self.format, stage, target, &out, &extra_args, &crates); - - // Don't open if the format is json - if let DocumentationFormat::Json = self.format { - return; - } - - if builder.paths.iter().any(|path| path.ends_with("library")) { - // For `x.py doc library --open`, open `std` by default. - let index = out.join("std").join("index.html"); - builder.open_in_browser(index); - } else { - for requested_crate in crates { - if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { - let index = out.join(requested_crate).join("index.html"); - builder.open_in_browser(index); - break; - } - } - } - } -} - -/// Name of the crates that are visible to consumers of the standard library. -/// Documentation for internal crates is handled by the rustc step, so internal crates will show -/// up there. -/// -/// Order here is important! -/// Crates need to be processed starting from the leaves, otherwise rustdoc will not -/// create correct links between crates because rustdoc depends on the -/// existence of the output directories to know if it should be a local -/// or remote link. -const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub enum DocumentationFormat { - Html, - Json, -} - -impl DocumentationFormat { - fn as_str(&self) -> &str { - match self { - DocumentationFormat::Html => "HTML", - DocumentationFormat::Json => "JSON", - } - } -} - -/// Build the documentation for public standard library crates. -fn doc_std( - builder: &Builder<'_>, - format: DocumentationFormat, - stage: u32, - target: TargetSelection, - out: &Path, - extra_args: &[&str], - requested_crates: &[String], -) { - let compiler = builder.compiler(stage, builder.config.build); - - let target_doc_dir_name = if format == DocumentationFormat::Json { "json-doc" } else { "doc" }; - let target_dir = builder.stage_out(compiler, Mode::Std).join(target).join(target_doc_dir_name); - - // This is directory where the compiler will place the output of the command. - // We will then copy the files from this directory into the final `out` directory, the specified - // as a function parameter. - let out_dir = target_dir.join(target).join("doc"); - - let mut cargo = - builder::Cargo::new(builder, compiler, Mode::Std, SourceType::InTree, target, Kind::Doc); - - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - cargo - .arg("--no-deps") - .arg("--target-dir") - .arg(&*target_dir.to_string_lossy()) - .arg("-Zskip-rustdoc-fingerprint") - .arg("-Zrustdoc-map") - .rustdocflag("--extern-html-root-url") - .rustdocflag("std_detect=https://docs.rs/std_detect/latest/") - .rustdocflag("--extern-html-root-takes-precedence") - .rustdocflag("--resource-suffix") - .rustdocflag(&builder.version); - for arg in extra_args { - cargo.rustdocflag(arg); - } - - if builder.config.library_docs_private_items { - cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); - } - - for krate in requested_crates { - if krate == "sysroot" { - // The sysroot crate is an implementation detail, don't include it in public docs. - continue; - } - cargo.arg("-p").arg(krate); - } - - let description = - format!("library{} in {} format", crate_description(requested_crates), format.as_str()); - let _guard = builder.msg_doc(compiler, description, target); - - cargo.into_cmd().run(builder); - builder.cp_link_r(&out_dir, out); -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub stage: u32, - pub target: TargetSelection, - crates: Vec, -} - -impl Rustc { - pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { stage, target, crates } - } -} - -impl Step for Rustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("rustc-main") - .path("compiler") - .default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustc { - stage: run.builder.top_stage, - target: run.target, - crates: run.make_run_crates(Alias::Compiler), - }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - // Build the standard library, so that proc-macros can use it. - // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, builder.config.build)); - - let _guard = builder.msg_sysroot_tool( - Kind::Doc, - stage, - format!("compiler{}", crate_description(&self.crates)), - compiler.host, - target, - ); - - // Build cargo command. - let mut cargo = builder::Cargo::new( - builder, - compiler, - Mode::Rustc, - SourceType::InTree, - target, - Kind::Doc, - ); - - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("-Znormalize-docs"); - cargo.rustdocflag("--show-type-layout"); - // FIXME: `--generate-link-to-definition` tries to resolve cfged out code - // see https://github.com/rust-lang/rust/pull/122066#issuecomment-1983049222 - // cargo.rustdocflag("--generate-link-to-definition"); - - compile::rustc_cargo(builder, &mut cargo, target, &compiler, &self.crates); - cargo.arg("-Zskip-rustdoc-fingerprint"); - - // Only include compiler crates, no dependencies of those, such as `libc`. - // Do link to dependencies on `docs.rs` however using `rustdoc-map`. - cargo.arg("--no-deps"); - cargo.arg("-Zrustdoc-map"); - - // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, - // once this is no longer an issue the special case for `ena` can be removed. - cargo.rustdocflag("--extern-html-root-url"); - cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); - - let mut to_open = None; - - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target).join("doc"); - for krate in &*self.crates { - // Create all crate output directories first to make sure rustdoc uses - // relative links. - // FIXME: Cargo should probably do this itself. - let dir_name = krate.replace('-', "_"); - t!(fs::create_dir_all(out_dir.join(&*dir_name))); - cargo.arg("-p").arg(krate); - if to_open.is_none() { - to_open = Some(dir_name); - } - } - - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. - // - // This is needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - - cargo.into_cmd().run(builder); - - if !builder.config.dry_run() { - // Sanity check on linked compiler crates - for krate in &*self.crates { - let dir_name = krate.replace('-', "_"); - // Making sure the directory exists and is not empty. - assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); - } - } - - if builder.paths.iter().any(|path| path.ends_with("compiler")) { - // For `x.py doc compiler --open`, open `rustc_middle` by default. - let index = out.join("rustc_middle").join("index.html"); - builder.open_in_browser(index); - } else if let Some(krate) = to_open { - // Let's open the first crate documentation page: - let index = out.join(krate).join("index.html"); - builder.open_in_browser(index); - } - } -} - -macro_rules! tool_doc { - ( - $tool: ident, - $path: literal, - $(rustc_tool = $rustc_tool:literal, )? - $(is_library = $is_library:expr,)? - $(crates = $crates:expr)? - $(, submodule $(= $submodule:literal)? )? - ) => { - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $tool { - target: TargetSelection, - } - - impl Step for $tool { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($tool { target: run.target }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let source_type = SourceType::InTree; - $( - let _ = source_type; // silence the "unused variable" warning - let source_type = SourceType::Submodule; - - let path = submodule_helper!( $path, submodule $( = $submodule )? ); - builder.require_submodule(path, None); - )? - - let stage = builder.top_stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, target)); - - if true $(&& $rustc_tool)? { - // Build rustc docs so that we generate relative links. - builder.ensure(Rustc::new(stage, target, builder)); - - // Rustdoc needs the rustc sysroot available to build. - // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed - // with strange errors, but only on a full bors test ... - builder.ensure(compile::Rustc::new(compiler, target)); - } - - // Build cargo command. - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - Kind::Doc, - $path, - source_type, - &[], - ); - - cargo.arg("-Zskip-rustdoc-fingerprint"); - // Only include compiler crates, no dependencies of those, such as `libc`. - cargo.arg("--no-deps"); - - if false $(|| $is_library)? { - cargo.arg("--lib"); - } - - $(for krate in $crates { - cargo.arg("-p").arg(krate); - })? - - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("--show-type-layout"); - // FIXME: `--generate-link-to-definition` tries to resolve cfged out code - // see https://github.com/rust-lang/rust/pull/122066#issuecomment-1983049222 - // cargo.rustdocflag("--generate-link-to-definition"); - - let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target).join("doc"); - $(for krate in $crates { - let dir_name = krate.replace("-", "_"); - t!(fs::create_dir_all(out_dir.join(&*dir_name))); - })? - - // Symlink compiler docs to the output directory of rustdoc documentation. - symlink_dir_force(&builder.config, &out, &out_dir); - let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - - let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); - cargo.into_cmd().run(builder); - - if !builder.config.dry_run() { - // Sanity check on linked doc directories - $(for krate in $crates { - let dir_name = krate.replace("-", "_"); - // Making sure the directory exists and is not empty. - assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); - })? - } - } - } - } -} - -// NOTE: make sure to register these in `Builder::get_step_description`. -tool_doc!( - BuildHelper, - "src/build_helper", - rustc_tool = false, - is_library = true, - crates = ["build_helper"] -); -tool_doc!(Rustdoc, "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); -tool_doc!(Rustfmt, "src/tools/rustfmt", crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"]); -tool_doc!(Clippy, "src/tools/clippy", crates = ["clippy_config", "clippy_utils"]); -tool_doc!(Miri, "src/tools/miri", crates = ["miri"]); -tool_doc!( - Cargo, - "src/tools/cargo", - rustc_tool = false, - crates = [ - "cargo", - "cargo-credential", - "cargo-platform", - "cargo-test-macro", - "cargo-test-support", - "cargo-util", - "cargo-util-schemas", - "crates-io", - "mdman", - "rustfix", - ], - submodule = "src/tools/cargo" -); -tool_doc!(Tidy, "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); -tool_doc!( - Bootstrap, - "src/bootstrap", - rustc_tool = false, - is_library = true, - crates = ["bootstrap"] -); -tool_doc!( - RunMakeSupport, - "src/tools/run-make-support", - rustc_tool = false, - is_library = true, - crates = ["run_make_support"] -); -tool_doc!( - Compiletest, - "src/tools/compiletest", - rustc_tool = false, - is_library = true, - crates = ["compiletest"] -); - -#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] -pub struct ErrorIndex { - pub target: TargetSelection, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/error_index_generator").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - let target = run.target; - run.builder.ensure(ErrorIndex { target }); - } - - /// Generates the HTML rendered error-index by running the - /// `error_index_generator` tool. - fn run(self, builder: &Builder<'_>) { - builder.info(&format!("Documenting error index ({})", self.target)); - let out = builder.doc_out(self.target); - t!(fs::create_dir_all(&out)); - tool::ErrorIndex::command(builder).arg("html").arg(out).arg(&builder.version).run(builder); - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBookGen { - target: TargetSelection, -} - -impl Step for UnstableBookGen { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBookGen { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.target; - - builder.info(&format!("Generating unstable book md files ({target})")); - let out = builder.md_doc_out(target).join("unstable-book"); - builder.create_dir(&out); - builder.remove_dir(&out); - let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); - cmd.arg(builder.src.join("library")); - cmd.arg(builder.src.join("compiler")); - cmd.arg(builder.src.join("src")); - cmd.arg(out); - - cmd.run(builder); - } -} - -fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { - if config.dry_run() { - return; - } - if let Ok(m) = fs::symlink_metadata(link) { - if m.file_type().is_dir() { - t!(fs::remove_dir_all(link)); - } else { - // handle directory junctions on windows by falling back to - // `remove_dir`. - t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); - } - } - - t!( - symlink_dir(config, original, link), - format!("failed to create link from {} -> {}", link.display(), original.display()) - ); -} - -#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustcBook { - pub compiler: Compiler, - pub target: TargetSelection, - pub validate: bool, -} - -impl Step for RustcBook { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/rustc").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - validate: false, - }); - } - - /// Builds the rustc book. - /// - /// The lints are auto-generated by a tool, and then merged into the book - /// in the "md-doc" directory in the build output directory. Then - /// "rustbook" is used to convert it to HTML. - fn run(self, builder: &Builder<'_>) { - let out_base = builder.md_doc_out(self.target).join("rustc"); - t!(fs::create_dir_all(&out_base)); - let out_listing = out_base.join("src/lints"); - builder.cp_link_r(&builder.src.join("src/doc/rustc"), &out_base); - builder.info(&format!("Generating lint docs ({})", self.target)); - - let rustc = builder.rustc(self.compiler); - // The tool runs `rustc` for extracting output examples, so it needs a - // functional sysroot. - builder.ensure(compile::Std::new(self.compiler, self.target)); - let mut cmd = builder.tool_cmd(Tool::LintDocs); - cmd.arg("--src"); - cmd.arg(builder.src.join("compiler")); - cmd.arg("--out"); - cmd.arg(&out_listing); - cmd.arg("--rustc"); - cmd.arg(&rustc); - cmd.arg("--rustc-target").arg(self.target.rustc_target_arg()); - if let Some(target_linker) = builder.linker(self.target) { - cmd.arg("--rustc-linker").arg(target_linker); - } - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if self.validate { - cmd.arg("--validate"); - } - // We need to validate nightly features, even on the stable channel. - // Set this unconditionally as the stage0 compiler may be being used to - // document. - cmd.env("RUSTC_BOOTSTRAP", "1"); - - // If the lib directories are in an unusual location (changed in - // config.toml), then this needs to explicitly update the dylib search - // path. - builder.add_rustc_lib_path(self.compiler, &mut cmd); - let doc_generator_guard = builder.msg( - Kind::Run, - self.compiler.stage, - "lint-docs", - self.compiler.host, - self.target, - ); - cmd.run(builder); - drop(doc_generator_guard); - - // Run rustbook/mdbook to generate the HTML pages. - builder.ensure(RustbookSrc { - target: self.target, - name: "rustc".to_owned(), - src: out_base, - parent: Some(self), - languages: vec![], - rustdoc_compiler: None, - }); - } -} - -#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] -pub struct Reference { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Reference { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/reference").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Reference { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Builds the reference book. - fn run(self, builder: &Builder<'_>) { - builder.require_submodule("src/doc/reference", None); - - // This is needed for generating links to the standard library using - // the mdbook-spec plugin. - builder.ensure(compile::Std::new(self.compiler, builder.config.build)); - - // Run rustbook/mdbook to generate the HTML pages. - builder.ensure(RustbookSrc { - target: self.target, - name: "reference".to_owned(), - src: builder.src.join("src/doc/reference"), - rustdoc_compiler: Some(self.compiler), - parent: Some(self), - languages: vec![], - }); - } -} diff --git a/standalonex/src/src/core/build_steps/format.rs b/standalonex/src/src/core/build_steps/format.rs deleted file mode 100644 index 5ca4321d..00000000 --- a/standalonex/src/src/core/build_steps/format.rs +++ /dev/null @@ -1,309 +0,0 @@ -//! Runs rustfmt on the repository. - -use std::collections::VecDeque; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::Mutex; -use std::sync::mpsc::SyncSender; - -use build_helper::ci::CiEnv; -use build_helper::git::get_git_modified_files; -use ignore::WalkBuilder; - -use crate::core::builder::Builder; -use crate::utils::exec::command; -use crate::utils::helpers::{self, program_out_of_date, t}; - -fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { - let mut cmd = Command::new(rustfmt); - // Avoid the submodule config paths from coming into play. We only allow a single global config - // for the workspace for now. - cmd.arg("--config-path").arg(src.canonicalize().unwrap()); - cmd.arg("--edition").arg("2021"); - cmd.arg("--unstable-features"); - cmd.arg("--skip-children"); - if check { - cmd.arg("--check"); - } - cmd.args(paths); - let cmd_debug = format!("{cmd:?}"); - let mut cmd = cmd.spawn().expect("running rustfmt"); - // Poor man's async: return a closure that might wait for rustfmt's completion (depending on - // the value of the `block` argument). - move |block: bool| -> bool { - let status = if !block { - match cmd.try_wait() { - Ok(Some(status)) => Ok(status), - Ok(None) => return false, - Err(err) => Err(err), - } - } else { - cmd.wait() - }; - if !status.unwrap().success() { - eprintln!( - "fmt error: Running `{}` failed.\nIf you're running `tidy`, \ - try again with `--bless`. Or, if you just want to format \ - code, run `./x.py fmt` instead.", - cmd_debug, - ); - crate::exit!(1); - } - true - } -} - -fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { - let stamp_file = build.out.join("rustfmt.stamp"); - - let mut cmd = command(match build.initial_rustfmt() { - Some(p) => p, - None => return None, - }); - cmd.arg("--version"); - - let output = cmd.allow_failure().run_capture(build); - if output.is_failure() { - return None; - } - Some((output.stdout(), stamp_file)) -} - -/// Return whether the format cache can be reused. -fn verify_rustfmt_version(build: &Builder<'_>) -> bool { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return false; - }; - !program_out_of_date(&stamp_file, &version) -} - -/// Updates the last rustfmt version used. -fn update_rustfmt_version(build: &Builder<'_>) { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return; - }; - t!(std::fs::write(stamp_file, version)) -} - -/// Returns the Rust files modified between the `merge-base` of HEAD and -/// rust-lang/master and what is now on the disk. Does not include removed files. -/// -/// Returns `None` if all files should be formatted. -fn get_modified_rs_files(build: &Builder<'_>) -> Result>, String> { - if !verify_rustfmt_version(build) { - return Ok(None); - } - - get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]) -} - -#[derive(serde_derive::Deserialize)] -struct RustfmtConfig { - ignore: Vec, -} - -// Prints output describing a collection of paths, with lines such as "formatted modified file -// foo/bar/baz" or "skipped 20 untracked files". -fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) { - let len = paths.len(); - let adjective = - if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() }; - if len <= 10 { - for path in paths { - println!("fmt: {verb} {adjective}file {path}"); - } - } else { - println!("fmt: {verb} {len} {adjective}files"); - } -} - -pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) { - if !paths.is_empty() { - eprintln!( - "fmt error: path arguments are no longer accepted; use `--all` to format everything" - ); - crate::exit!(1); - }; - if build.config.dry_run() { - return; - } - - // By default, we only check modified files locally to speed up runtime. Exceptions are if - // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in - // `get_modified_rs_files` letting regressions slip through; we also care about CI time less - // since this is still very fast compared to building the compiler. - let all = all || CiEnv::is_ci(); - - let mut builder = ignore::types::TypesBuilder::new(); - builder.add_defaults(); - builder.select("rust"); - let matcher = builder.build().unwrap(); - let rustfmt_config = build.src.join("rustfmt.toml"); - if !rustfmt_config.exists() { - eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist."); - eprintln!("fmt error: This may happen in distributed tarballs."); - return; - } - let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); - let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); - let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src); - for ignore in rustfmt_config.ignore { - if ignore.starts_with('!') { - // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`, - // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an - // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse - // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine - // explicit whitelisted entries and traversal of unmentioned files, but for now just - // forbid such entries. - eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry"); - crate::exit!(1); - } else { - override_builder.add(&format!("!{ignore}")).expect(&ignore); - } - } - let git_available = - helpers::git(None).allow_failure().arg("--version").run_capture(build).is_success(); - - let mut adjective = None; - if git_available { - let in_working_tree = helpers::git(Some(&build.src)) - .allow_failure() - .arg("rev-parse") - .arg("--is-inside-work-tree") - .run_capture(build) - .is_success(); - if in_working_tree { - let untracked_paths_output = helpers::git(Some(&build.src)) - .arg("status") - .arg("--porcelain") - .arg("-z") - .arg("--untracked-files=normal") - .run_capture_stdout(build) - .stdout(); - let untracked_paths: Vec<_> = untracked_paths_output - .split_terminator('\0') - .filter_map( - |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match - ) - .map(|x| x.to_string()) - .collect(); - print_paths("skipped", Some("untracked"), &untracked_paths); - - for untracked_path in untracked_paths { - // The leading `/` makes it an exact match against the - // repository root, rather than a glob. Without that, if you - // have `foo.rs` in the repository root it will also match - // against anything like `compiler/rustc_foo/src/foo.rs`, - // preventing the latter from being formatted. - override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path); - } - if !all { - adjective = Some("modified"); - match get_modified_rs_files(build) { - Ok(Some(files)) => { - if files.is_empty() { - println!("fmt info: No modified files detected for formatting."); - return; - } - - for file in files { - override_builder.add(&format!("/{file}")).expect(&file); - } - } - Ok(None) => {} - Err(err) => { - eprintln!("fmt warning: Something went wrong running git commands:"); - eprintln!("fmt warning: {err}"); - eprintln!("fmt warning: Falling back to formatting all files."); - } - } - } - } else { - eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks"); - } - } else { - eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks"); - } - - let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword - - let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { - eprintln!("fmt error: `x fmt` is not supported on this channel"); - crate::exit!(1); - }); - assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); - let src = build.src.clone(); - let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); - let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel(); - - // There is a lot of blocking involved in spawning a child process and reading files to format. - // Spawn more processes than available concurrency to keep the CPU busy. - let max_processes = build.jobs() as usize * 2; - - // Spawn child processes on a separate thread so we can batch entries we have received from - // ignore. - let thread = std::thread::spawn(move || { - let mut children = VecDeque::new(); - while let Ok(path) = rx.recv() { - // Try getting more paths from the channel to amortize the overhead of spawning - // processes. - let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect(); - - let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); - children.push_back(child); - - // Poll completion before waiting. - for i in (0..children.len()).rev() { - if children[i](false) { - children.swap_remove_back(i); - break; - } - } - - if children.len() >= max_processes { - // Await oldest child. - children.pop_front().unwrap()(true); - } - } - - // Await remaining children. - for mut child in children { - child(true); - } - }); - - let formatted_paths = Mutex::new(Vec::new()); - let formatted_paths_ref = &formatted_paths; - walker.run(|| { - let tx = tx.clone(); - Box::new(move |entry| { - let cwd = std::env::current_dir(); - let entry = t!(entry); - if entry.file_type().map_or(false, |t| t.is_file()) { - formatted_paths_ref.lock().unwrap().push({ - // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter - // relative path. - let mut path = entry.clone().into_path(); - if let Ok(cwd) = cwd { - if let Ok(path2) = path.strip_prefix(cwd) { - path = path2.to_path_buf(); - } - } - path.display().to_string() - }); - t!(tx.send(entry.into_path())); - } - ignore::WalkState::Continue - }) - }); - let mut paths = formatted_paths.into_inner().unwrap(); - paths.sort(); - print_paths(if check { "checked" } else { "formatted" }, adjective, &paths); - - drop(tx); - - thread.join().unwrap(); - if !check { - update_rustfmt_version(build); - } -} diff --git a/standalonex/src/src/core/build_steps/gcc.rs b/standalonex/src/src/core/build_steps/gcc.rs deleted file mode 100644 index b950bec1..00000000 --- a/standalonex/src/src/core/build_steps/gcc.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Compilation of native dependencies like GCC. -//! -//! Native projects like GCC unfortunately aren't suited just yet for -//! compilation in build scripts that Cargo has. This is because the -//! compilation takes a *very* long time but also because we don't want to -//! compile GCC 3 times as part of a normal bootstrap (we want it cached). -//! -//! GCC and compiler-rt are essentially just wired up to everything else to -//! ensure that they're always in place if needed. - -use std::fs; -use std::path::PathBuf; -use std::sync::OnceLock; - -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::core::config::TargetSelection; -use crate::utils::exec::command; -use crate::utils::helpers::{self, HashStamp, t}; -use crate::{Kind, generate_smart_stamp_hash}; - -pub struct Meta { - stamp: HashStamp, - out_dir: PathBuf, - install_dir: PathBuf, - root: PathBuf, -} - -pub enum GccBuildStatus { - AlreadyBuilt, - ShouldBuild(Meta), -} - -/// This returns whether we've already previously built GCC. -/// -/// It's used to avoid busting caches during x.py check -- if we've already built -/// GCC, it's fine for us to not try to avoid doing so. -pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus { - // Initialize the gcc submodule if not initialized already. - builder.config.update_submodule("src/gcc"); - - // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI. - // builder.config.maybe_download_ci_gcc(); - - let root = builder.src.join("src/gcc"); - let out_dir = builder.gcc_out(target).join("build"); - let install_dir = builder.gcc_out(target).join("install"); - - static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - generate_smart_stamp_hash( - builder, - &builder.config.src.join("src/gcc"), - builder.in_tree_gcc_info.sha().unwrap_or_default(), - ) - }); - - let stamp = out_dir.join("gcc-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info( - "Could not determine the GCC submodule commit hash. \ - Assuming that an GCC rebuild is not necessary.", - ); - builder.info(&format!( - "To force GCC to rebuild, remove the file `{}`", - stamp.path.display() - )); - } - return GccBuildStatus::AlreadyBuilt; - } - - GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root }) -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Gcc { - pub target: TargetSelection, -} - -impl Step for Gcc { - type Output = bool; - - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/gcc").alias("gcc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Gcc { target: run.target }); - } - - /// Compile GCC for `target`. - fn run(self, builder: &Builder<'_>) -> bool { - let target = self.target; - - // If GCC has already been built, we avoid building it again. - let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target) - { - GccBuildStatus::AlreadyBuilt => return true, - GccBuildStatus::ShouldBuild(m) => m, - }; - - let _guard = builder.msg_unstaged(Kind::Build, "GCC", target); - t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - if builder.config.dry_run() { - return true; - } - - command(root.join("contrib/download_prerequisites")).current_dir(&root).run(builder); - command(root.join("configure")) - .current_dir(&out_dir) - .arg("--enable-host-shared") - .arg("--enable-languages=jit") - .arg("--enable-checking=release") - .arg("--disable-bootstrap") - .arg("--disable-multilib") - .arg(format!("--prefix={}", install_dir.display())) - .run(builder); - command("make").current_dir(&out_dir).arg(format!("-j{}", builder.jobs())).run(builder); - command("make").current_dir(&out_dir).arg("install").run(builder); - - let lib_alias = install_dir.join("lib/libgccjit.so.0"); - if !lib_alias.exists() { - t!(builder.symlink_file(install_dir.join("lib/libgccjit.so"), lib_alias,)); - } - - t!(stamp.write()); - - true - } -} diff --git a/standalonex/src/src/core/build_steps/install.rs b/standalonex/src/src/core/build_steps/install.rs deleted file mode 100644 index 0ce86ead..00000000 --- a/standalonex/src/src/core/build_steps/install.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! Implementation of the install aspects of the compiler. -//! -//! This module is responsible for installing the standard library, -//! compiler, and documentation. - -use std::path::{Component, Path, PathBuf}; -use std::{env, fs}; - -use crate::core::build_steps::dist; -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::core::config::{Config, TargetSelection}; -use crate::utils::exec::command; -use crate::utils::helpers::t; -use crate::utils::tarball::GeneratedTarball; -use crate::{Compiler, Kind}; - -#[cfg(target_os = "illumos")] -const SHELL: &str = "bash"; -#[cfg(not(target_os = "illumos"))] -const SHELL: &str = "sh"; - -/// We have to run a few shell scripts, which choke quite a bit on both `\` -/// characters and on `C:\` paths, so normalize both of them away. -fn sanitize_sh(path: &Path) -> String { - let path = path.to_str().unwrap().replace('\\', "/"); - return change_drive(unc_to_lfs(&path)).unwrap_or(path); - - fn unc_to_lfs(s: &str) -> &str { - s.strip_prefix("//?/").unwrap_or(s) - } - - fn change_drive(s: &str) -> Option { - let mut ch = s.chars(); - let drive = ch.next().unwrap_or('C'); - if ch.next() != Some(':') { - return None; - } - if ch.next() != Some('/') { - return None; - } - Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) - } -} - -fn is_dir_writable_for_user(dir: &Path) -> bool { - let tmp = dir.join(".tmp"); - match fs::create_dir_all(&tmp) { - Ok(_) => { - fs::remove_dir_all(tmp).unwrap(); - true - } - Err(e) => { - if e.kind() == std::io::ErrorKind::PermissionDenied { - false - } else { - panic!("Failed the write access check for the current user. {}", e); - } - } - } -} - -fn install_sh( - builder: &Builder<'_>, - package: &str, - stage: u32, - host: Option, - tarball: &GeneratedTarball, -) { - let _guard = builder.msg(Kind::Install, stage, package, host, host); - - let prefix = default_path(&builder.config.prefix, "/usr/local"); - let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); - let destdir_env = env::var_os("DESTDIR").map(PathBuf::from); - - // Sanity checks on the write access of user. - // - // When the `DESTDIR` environment variable is present, there is no point to - // check write access for `prefix` and `sysconfdir` individually, as they - // are combined with the path from the `DESTDIR` environment variable. In - // this case, we only need to check the `DESTDIR` path, disregarding the - // `prefix` and `sysconfdir` paths. - if let Some(destdir) = &destdir_env { - assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR."); - } else { - assert!( - is_dir_writable_for_user(&prefix), - "User doesn't have write access on `install.prefix` path in the `config.toml`.", - ); - assert!( - is_dir_writable_for_user(&sysconfdir), - "User doesn't have write access on `install.sysconfdir` path in `config.toml`." - ); - } - - let datadir = prefix.join(default_path(&builder.config.datadir, "share")); - let docdir = prefix.join(default_path(&builder.config.docdir, &format!("share/doc/{package}"))); - let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); - let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); - let bindir = prefix.join(&builder.config.bindir); // Default in config.rs - - let empty_dir = builder.out.join("tmp/empty_dir"); - t!(fs::create_dir_all(&empty_dir)); - - let mut cmd = command(SHELL); - cmd.current_dir(&empty_dir) - .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) - .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix))) - .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir))) - .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir))) - .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir))) - .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir))) - .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir))) - .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir))) - .arg("--disable-ldconfig"); - cmd.run(builder); - t!(fs::remove_dir_all(&empty_dir)); -} - -fn default_path(config: &Option, default: &str) -> PathBuf { - config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) -} - -fn prepare_dir(destdir_env: &Option, mut path: PathBuf) -> String { - // The DESTDIR environment variable is a standard way to install software in a subdirectory - // while keeping the original directory structure, even if the prefix or other directories - // contain absolute paths. - // - // More information on the environment variable is available here: - // https://www.gnu.org/prep/standards/html_node/DESTDIR.html - if let Some(destdir) = destdir_env { - let without_destdir = path.clone(); - path.clone_from(destdir); - // Custom .join() which ignores disk roots. - for part in without_destdir.components() { - if let Component::Normal(s) = part { - path.push(s) - } - } - } - - // The installation command is not executed from the current directory, but from a temporary - // directory. To prevent relative paths from breaking this converts relative paths to absolute - // paths. std::fs::canonicalize is not used as that requires the path to actually be present. - if path.is_relative() { - path = std::env::current_dir().expect("failed to get the current directory").join(path); - assert!(path.is_absolute(), "could not make the path relative"); - } - - sanitize_sh(&path) -} - -macro_rules! install { - (($sel:ident, $builder:ident, $_config:ident), - $($name:ident, - $condition_name: ident = $path_or_alias: literal, - $default_cond:expr, - only_hosts: $only_hosts:expr, - $run_item:block $(, $c:ident)*;)+) => { - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl $name { - #[allow(dead_code)] - fn should_build(config: &Config) -> bool { - config.extended && config.tools.as_ref() - .map_or(true, |t| t.contains($path_or_alias)) - } - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = $only_hosts; - $(const $c: bool = true;)* - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let $_config = &run.builder.config; - run.$condition_name($path_or_alias).default_condition($default_cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run($sel, $builder: &Builder<'_>) { - $run_item - } - })+ - } -} - -install!((self, builder, _config), - Docs, path = "src/doc", _config.docs, only_hosts: false, { - let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); - install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); - }; - Std, path = "library/std", true, only_hosts: false, { - // `expect` should be safe, only None when host != build, but this - // only runs when host == build - let tarball = builder.ensure(dist::Std { - compiler: self.compiler, - target: self.target - }).expect("missing std"); - install_sh(builder, "std", self.compiler.stage, Some(self.target), &tarball); - }; - Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) - .expect("missing cargo"); - install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); - }; - RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = - builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) - { - install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) - .expect("missing clippy"); - install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); - }; - Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { - install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); - } else { - // Miri is only available on nightly - builder.info( - &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { - install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), - ); - } - }; - Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Rustfmt { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - Rustc, path = "compiler/rustc", true, only_hosts: true, { - let tarball = builder.ensure(dist::Rustc { - compiler: builder.compiler(builder.top_stage, self.target), - }); - install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); - }; - RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::CodegenBackend { - compiler: self.compiler, - backend: "cranelift".to_string(), - }) { - install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})", - self.compiler.stage, self.target), - ); - } - }; - LlvmBitcodeLinker, alias = "llvm-bitcode-linker", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::LlvmBitcodeLinker { compiler: self.compiler, target: self.target }) { - install_sh(builder, "llvm-bitcode-linker", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping llvm-bitcode-linker stage{} ({})", self.compiler.stage, self.target), - ); - } - }; -); - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Src { - pub stage: u32, -} - -impl Step for Src { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let config = &run.builder.config; - let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); - run.path("src").default_condition(cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src { stage: run.builder.top_stage }); - } - - fn run(self, builder: &Builder<'_>) { - let tarball = builder.ensure(dist::Src); - install_sh(builder, "src", self.stage, None, &tarball); - } -} diff --git a/standalonex/src/src/core/build_steps/llvm.rs b/standalonex/src/src/core/build_steps/llvm.rs deleted file mode 100644 index 9734a0dc..00000000 --- a/standalonex/src/src/core/build_steps/llvm.rs +++ /dev/null @@ -1,1524 +0,0 @@ -//! Compilation of native dependencies like LLVM. -//! -//! Native projects like LLVM unfortunately aren't suited just yet for -//! compilation in build scripts that Cargo has. This is because the -//! compilation takes a *very* long time but also because we don't want to -//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). -//! -//! LLVM and compiler-rt are essentially just wired up to everything else to -//! ensure that they're always in place if needed. - -use std::env; -use std::env::consts::EXE_EXTENSION; -use std::ffi::{OsStr, OsString}; -use std::fs::{self, File}; -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; - -use build_helper::ci::CiEnv; -use build_helper::git::get_closest_merge_commit; - -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::core::config::{Config, TargetSelection}; -use crate::utils::exec::command; -use crate::utils::helpers::{ - self, HashStamp, exe, get_clang_cl_resource_dir, t, unhashed_basename, up_to_date, -}; -use crate::{CLang, GitRepo, Kind, generate_smart_stamp_hash}; - -#[derive(Clone)] -pub struct LlvmResult { - /// Path to llvm-config binary. - /// NB: This is always the host llvm-config! - pub llvm_config: PathBuf, - /// Path to LLVM cmake directory for the target. - pub llvm_cmake_dir: PathBuf, -} - -pub struct Meta { - stamp: HashStamp, - res: LlvmResult, - out_dir: PathBuf, - root: String, -} - -pub enum LlvmBuildStatus { - AlreadyBuilt(LlvmResult), - ShouldBuild(Meta), -} - -impl LlvmBuildStatus { - pub fn should_build(&self) -> bool { - match self { - LlvmBuildStatus::AlreadyBuilt(_) => false, - LlvmBuildStatus::ShouldBuild(_) => true, - } - } -} - -/// Linker flags to pass to LLVM's CMake invocation. -#[derive(Debug, Clone, Default)] -struct LdFlags { - /// CMAKE_EXE_LINKER_FLAGS - exe: OsString, - /// CMAKE_SHARED_LINKER_FLAGS - shared: OsString, - /// CMAKE_MODULE_LINKER_FLAGS - module: OsString, -} - -impl LdFlags { - fn push_all(&mut self, s: impl AsRef) { - let s = s.as_ref(); - self.exe.push(" "); - self.exe.push(s); - self.shared.push(" "); - self.shared.push(s); - self.module.push(" "); - self.module.push(s); - } -} - -/// This returns whether we've already previously built LLVM. -/// -/// It's used to avoid busting caches during x.py check -- if we've already built -/// LLVM, it's fine for us to not try to avoid doing so. -/// -/// This will return the llvm-config if it can get it (but it will not build it -/// if not). -pub fn prebuilt_llvm_config( - builder: &Builder<'_>, - target: TargetSelection, - // Certain commands (like `x test mir-opt --bless`) may call this function with different targets, - // which could bypass the CI LLVM early-return even if `builder.config.llvm_from_ci` is true. - // This flag should be `true` only if the caller needs the LLVM sources (e.g., if it will build LLVM). - handle_submodule_when_needed: bool, -) -> LlvmBuildStatus { - builder.config.maybe_download_ci_llvm(); - - // If we're using a custom LLVM bail out here, but we can only use a - // custom LLVM for the build triple. - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref s) = config.llvm_config { - check_llvm_version(builder, s); - let llvm_config = s.to_path_buf(); - let mut llvm_cmake_dir = llvm_config.clone(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.push("lib"); - llvm_cmake_dir.push("cmake"); - llvm_cmake_dir.push("llvm"); - return LlvmBuildStatus::AlreadyBuilt(LlvmResult { llvm_config, llvm_cmake_dir }); - } - } - - if handle_submodule_when_needed { - // If submodules are disabled, this does nothing. - builder.config.update_submodule("src/llvm-project"); - } - - let root = "src/llvm-project/llvm"; - let out_dir = builder.llvm_out(target); - - let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); - llvm_config_ret_dir.push("bin"); - let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); - let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); - let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; - - static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - generate_smart_stamp_hash( - builder, - &builder.config.src.join("src/llvm-project"), - builder.in_tree_llvm_info.sha().unwrap_or_default(), - ) - }); - - let stamp = out_dir.join("llvm-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info( - "Could not determine the LLVM submodule commit hash. \ - Assuming that an LLVM rebuild is not necessary.", - ); - builder.info(&format!( - "To force LLVM to rebuild, remove the file `{}`", - stamp.path.display() - )); - } - return LlvmBuildStatus::AlreadyBuilt(res); - } - - LlvmBuildStatus::ShouldBuild(Meta { stamp, res, out_dir, root: root.into() }) -} - -/// This retrieves the LLVM sha we *want* to use, according to git history. -pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { - let llvm_sha = if is_git { - get_closest_merge_commit(Some(&config.src), &config.git_config(), &[ - config.src.join("src/llvm-project"), - config.src.join("src/bootstrap/download-ci-llvm-stamp"), - // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - config.src.join("src/version"), - ]) - .unwrap() - } else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) { - info.sha.trim().to_owned() - } else { - "".to_owned() - }; - - if llvm_sha.is_empty() { - eprintln!("error: could not find commit hash for downloading LLVM"); - eprintln!("HELP: maybe your repository history is too shallow?"); - eprintln!("HELP: consider disabling `download-ci-llvm`"); - eprintln!("HELP: or fetch enough history to include one upstream commit"); - panic!(); - } - - llvm_sha -} - -/// Returns whether the CI-found LLVM is currently usable. -/// -/// This checks both the build triple platform to confirm we're usable at all, -/// and then verifies if the current HEAD matches the detected LLVM SHA head, -/// in which case LLVM is indicated as not available. -pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { - // This is currently all tier 1 targets and tier 2 targets with host tools - // (since others may not have CI artifacts) - // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - let supported_platforms = [ - // tier 1 - ("aarch64-unknown-linux-gnu", false), - ("aarch64-apple-darwin", false), - ("i686-pc-windows-gnu", false), - ("i686-pc-windows-msvc", false), - ("i686-unknown-linux-gnu", false), - // ("x86_64-unknown-linux-gnu", true), - ("x86_64-apple-darwin", true), - ("x86_64-pc-windows-gnu", true), - ("x86_64-pc-windows-msvc", true), - // tier 2 with host tools - ("aarch64-pc-windows-msvc", false), - ("aarch64-unknown-linux-musl", false), - ("arm-unknown-linux-gnueabi", false), - ("arm-unknown-linux-gnueabihf", false), - ("armv7-unknown-linux-gnueabihf", false), - ("loongarch64-unknown-linux-gnu", false), - ("loongarch64-unknown-linux-musl", false), - ("mips-unknown-linux-gnu", false), - ("mips64-unknown-linux-gnuabi64", false), - ("mips64el-unknown-linux-gnuabi64", false), - ("mipsel-unknown-linux-gnu", false), - ("powerpc-unknown-linux-gnu", false), - ("powerpc64-unknown-linux-gnu", false), - ("powerpc64le-unknown-linux-gnu", false), - ("riscv64gc-unknown-linux-gnu", false), - ("s390x-unknown-linux-gnu", false), - ("x86_64-unknown-freebsd", false), - ("x86_64-unknown-illumos", false), - ("x86_64-unknown-linux-musl", false), - ("x86_64-unknown-netbsd", false), - ]; - - if !supported_platforms.contains(&(&*config.build.triple, asserts)) - && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) - { - return false; - } - - if is_ci_llvm_modified(config) { - eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); - return false; - } - - true -} - -/// Returns true if we're running in CI with modified LLVM (and thus can't download it) -pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - // If not running in a CI environment, return false. - if !CiEnv::is_ci() { - return false; - } - - // In rust-lang/rust managed CI, assert the existence of the LLVM submodule. - if CiEnv::is_rust_lang_managed_ci_job() { - assert!( - config.in_tree_llvm_info.is_managed_git_subrepository(), - "LLVM submodule must be fetched in rust-lang/rust managed CI builders." - ); - } - // If LLVM submodule isn't present, skip the change check as it won't work. - else if !config.in_tree_llvm_info.is_managed_git_subrepository() { - return false; - } - - let llvm_sha = detect_llvm_sha(config, true); - let head_sha = crate::output( - helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD").as_command_mut(), - ); - let head_sha = head_sha.trim(); - llvm_sha == head_sha -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Llvm { - pub target: TargetSelection, -} - -impl Step for Llvm { - type Output = LlvmResult; - - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project").path("src/llvm-project/llvm") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Llvm { target: run.target }); - } - - /// Compile LLVM for `target`. - fn run(self, builder: &Builder<'_>) -> LlvmResult { - let target = self.target; - let target_native = if self.target.starts_with("riscv") { - // RISC-V target triples in Rust is not named the same as C compiler target triples. - // This converts Rust RISC-V target triples to C compiler triples. - let idx = target.triple.find('-').unwrap(); - - format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) - } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { - // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. - // Set the version suffix to 13.0 so the correct target details are used. - format!("{}{}", self.target, "13.0") - } else { - target.to_string() - }; - - // If LLVM has already been built or been downloaded through download-ci-llvm, we avoid building it again. - let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target, true) { - LlvmBuildStatus::AlreadyBuilt(p) => return p, - LlvmBuildStatus::ShouldBuild(m) => m, - }; - - if builder.llvm_link_shared() && target.is_windows() { - panic!("shared linking to LLVM is not currently supported on {}", target.triple); - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); - t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - // https://llvm.org/docs/CMake.html - let mut cfg = cmake::Config::new(builder.src.join(root)); - let mut ldflags = LdFlags::default(); - - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - // NOTE: remember to also update `config.example.toml` when changing the - // defaults! - let llvm_targets = match &builder.config.llvm_targets { - Some(s) => s, - None => { - "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ - SBF;Sparc;SystemZ;WebAssembly;X86" - } - }; - - let llvm_exp_targets = match builder.config.llvm_experimental_targets { - Some(ref s) => s, - None => "AVR;M68k;CSKY;Xtensa", - }; - - let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; - let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; - let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; - let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "ON") - .define("LLVM_ENABLE_ASSERTIONS", assertions) - .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") - .define("LLVM_ENABLE_PLUGINS", plugins) - .define("LLVM_TARGETS_TO_BUILD", llvm_targets) - .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) - .define("LLVM_INCLUDE_EXAMPLES", "OFF") - .define("LLVM_INCLUDE_DOCS", "OFF") - .define("LLVM_INCLUDE_BENCHMARKS", "OFF") - .define("LLVM_INCLUDE_TESTS", enable_tests) - // FIXME: remove this when minimal llvm is 19 - .define("LLVM_ENABLE_TERMINFO", "OFF") - .define("LLVM_ENABLE_LIBEDIT", "OFF") - .define("LLVM_ENABLE_BINDINGS", "OFF") - .define("LLVM_ENABLE_Z3_SOLVER", "OFF") - .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) - .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) - .define("LLVM_ENABLE_WARNINGS", enable_warnings); - - // Parts of our test suite rely on the `FileCheck` tool, which is built by default in - // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. - // This flag makes sure `FileCheck` is copied in the final binaries directory. - cfg.define("LLVM_INSTALL_UTILS", "ON"); - - if builder.config.llvm_profile_generate { - cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); - if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { - cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); - } - cfg.define("LLVM_BUILD_RUNTIME", "No"); - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - cfg.define("LLVM_PROFDATA_FILE", path); - } - - // Libraries for ELF section compression. - if !target.is_windows() { - cfg.define("LLVM_ENABLE_ZLIB", "ON"); - } else { - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // Are we compiling for iOS/tvOS/watchOS/visionOS? - if target.contains("apple-ios") - || target.contains("apple-tvos") - || target.contains("apple-watchos") - || target.contains("apple-visionos") - { - // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. - cfg.define("CMAKE_OSX_SYSROOT", "/"); - cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); - // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. - cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); - // Zlib fails to link properly, leading to a compiler error. - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // This setting makes the LLVM tools link to the dynamic LLVM library, - // which saves both memory during parallel links and overall disk space - // for the tools. We don't do this on every platform as it doesn't work - // equally well everywhere. - if builder.llvm_link_shared() { - cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); - } else { - cfg.define("LIBCLANG_BUILD_STATIC", "ON"); - cfg.define("CLANG_LINK_CLANG_DYLIB", "OFF"); - cfg.define("LLVM_BUILD_LLVM_DYLIB", "OFF"); - cfg.define("LLVM_LINK_LLVM_DYLIB", "OFF"); - } - - if (target.starts_with("csky") - || target.starts_with("riscv") - || target.starts_with("sparc-")) - && !target.contains("freebsd") - && !target.contains("openbsd") - && !target.contains("netbsd") - { - // CSKY and RISC-V GCC erroneously requires linking against - // `libatomic` when using 1-byte and 2-byte C++ - // atomics but the LLVM build system check cannot - // detect this. Therefore it is set manually here. - // Some BSD uses Clang as its system compiler and - // provides no libatomic in its base system so does - // not want this. 32-bit SPARC requires linking against - // libatomic as well. - ldflags.exe.push(" -latomic"); - ldflags.shared.push(" -latomic"); - } - - if target.starts_with("mips") && target.contains("netbsd") { - // LLVM wants 64-bit atomics, while mipsel is 32-bit only, so needs -latomic - ldflags.exe.push(" -latomic"); - ldflags.shared.push(" -latomic"); - } - - if target.is_msvc() { - cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", "MultiThreaded"); - cfg.static_crt(true); - } - - if target.starts_with("i686") { - cfg.define("LLVM_BUILD_32_BITS", "ON"); - } - - let mut enabled_llvm_projects = Vec::new(); - - if helpers::forcing_clang_based_tests() { - enabled_llvm_projects.push("clang"); - enabled_llvm_projects.push("compiler-rt"); - } - - if builder.config.llvm_polly { - enabled_llvm_projects.push("polly"); - } - - if builder.config.llvm_clang { - enabled_llvm_projects.push("clang"); - } - - if let Some(projects) = &builder.config.llvm_enable_projects { - for p in projects.split(';') { - enabled_llvm_projects.push(p); - } - } - - // We want libxml to be disabled. - // See https://github.com/rust-lang/rust/pull/50104 - cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); - - if !enabled_llvm_projects.is_empty() { - enabled_llvm_projects.sort(); - enabled_llvm_projects.dedup(); - cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); - } - - let mut enabled_llvm_runtimes = Vec::new(); - - if builder.config.llvm_offload { - enabled_llvm_runtimes.push("offload"); - //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. - //Remove this line once they achieved it. - enabled_llvm_runtimes.push("openmp"); - } - - if !enabled_llvm_runtimes.is_empty() { - enabled_llvm_runtimes.sort(); - enabled_llvm_runtimes.dedup(); - cfg.define("LLVM_ENABLE_RUNTIMES", enabled_llvm_runtimes.join(";")); - } - - if let Some(num_linkers) = builder.config.llvm_link_jobs { - if num_linkers > 0 { - cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); - } - } - - // https://llvm.org/docs/HowToCrossCompileLLVM.html - if target != builder.config.build { - let LlvmResult { llvm_config, .. } = - builder.ensure(Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_bindir = - command(&llvm_config).arg("--bindir").run_capture_stdout(builder).stdout(); - let host_bin = Path::new(llvm_bindir.trim()); - cfg.define( - "LLVM_TABLEGEN", - host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - // LLVM_NM is required for cross compiling using MSVC - cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); - } - cfg.define("LLVM_CONFIG_PATH", llvm_config); - if builder.config.llvm_clang { - let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); - let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); - if !builder.config.dry_run() && !clang_tblgen.exists() { - panic!("unable to find {}", clang_tblgen.display()); - } - cfg.define("CLANG_TABLEGEN", clang_tblgen); - } - } - - let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { - // Allow version-suffix="" to not define a version suffix at all. - if !suffix.is_empty() { Some(suffix.to_string()) } else { None } - } else if builder.config.channel == "dev" { - // Changes to a version suffix require a complete rebuild of the LLVM. - // To avoid rebuilds during a time of version bump, don't include rustc - // release number on the dev channel. - Some("-rust-dev".to_string()) - } else { - Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) - }; - if let Some(ref suffix) = llvm_version_suffix { - cfg.define("LLVM_VERSION_SUFFIX", suffix); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - for (key, val) in &builder.config.llvm_build_config { - cfg.define(key, val); - } - - if builder.config.dry_run() { - return res; - } - - cfg.build(); - - // Helper to find the name of LLVM's shared library on darwin and linux. - let find_llvm_lib_name = |extension| { - let version = - command(&res.llvm_config).arg("--version").run_capture_stdout(builder).stdout(); - let major = version.split('.').next().unwrap(); - - match &llvm_version_suffix { - Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), - None => format!("libLLVM-{major}.{extension}"), - } - }; - - // FIXME(ZuseZ4): Do we need that for Enzyme too? - // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned - // libLLVM.dylib will be built. However, llvm-config will still look - // for a versioned path like libLLVM-14.dylib. Manually create a symbolic - // link to make llvm-config happy. - if builder.llvm_link_shared() && target.contains("apple-darwin") { - let lib_name = find_llvm_lib_name("dylib"); - let lib_llvm = out_dir.join("build").join("lib").join(lib_name); - if !lib_llvm.exists() { - t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); - } - } - - // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: - // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to - // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. - if builder.llvm_link_shared() - && builder.config.llvm_optimize - && !builder.config.llvm_release_debuginfo - { - // Find the name of the LLVM shared library that we just built. - let lib_name = find_llvm_lib_name("so"); - - // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its - // debuginfo. - crate::core::build_steps::compile::strip_debug( - builder, - target, - &out_dir.join("lib").join(&lib_name), - ); - crate::core::build_steps::compile::strip_debug( - builder, - target, - &out_dir.join("build").join("lib").join(&lib_name), - ); - } - - t!(stamp.write()); - - res - } -} - -fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { - if builder.config.dry_run() { - return; - } - - let version = command(llvm_config).arg("--version").run_capture_stdout(builder).stdout(); - let mut parts = version.split('.').take(2).filter_map(|s| s.parse::().ok()); - if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { - if major >= 18 { - return; - } - } - panic!("\n\nbad LLVM version: {version}, need >=18\n\n") -} - -fn configure_cmake( - builder: &Builder<'_>, - target: TargetSelection, - cfg: &mut cmake::Config, - use_compiler_launcher: bool, - mut ldflags: LdFlags, - suppressed_compiler_flag_prefixes: &[&str], -) { - // Do not print installation messages for up-to-date files. - // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. - cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); - - // Do not allow the user's value of DESTDIR to influence where - // LLVM will install itself. LLVM must always be installed in our - // own build directories. - cfg.env("DESTDIR", ""); - - if builder.ninja() { - cfg.generator("Ninja"); - } - cfg.target(&target.triple).host(&builder.config.build.triple); - - if target != builder.config.build { - cfg.define("CMAKE_CROSSCOMPILING", "True"); - - if target.contains("netbsd") { - cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); - } else if target.contains("dragonfly") { - cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); - } else if target.contains("freebsd") { - cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); - } else if target.is_windows() { - cfg.define("CMAKE_SYSTEM_NAME", "Windows"); - } else if target.contains("haiku") { - cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); - } else if target.contains("solaris") || target.contains("illumos") { - cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); - } else if target.contains("linux") { - cfg.define("CMAKE_SYSTEM_NAME", "Linux"); - } else { - builder.info(&format!( - "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", - )); - } - - // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in - // that case like CMake we cannot easily determine system version either. - // - // Since, the LLVM itself makes rather limited use of version checks in - // CMakeFiles (and then only in tests), and so far no issues have been - // reported, the system version is currently left unset. - - if target.contains("darwin") { - // Make sure that CMake does not build universal binaries on macOS. - // Explicitly specify the one single target architecture. - if target.starts_with("aarch64") { - // macOS uses a different name for building arm64 - cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); - } else if target.starts_with("i686") { - // macOS uses a different name for building i386 - cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); - } else { - cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); - } - } - } - - let sanitize_cc = |cc: &Path| { - if target.is_msvc() { - OsString::from(cc.to_str().unwrap().replace('\\', "/")) - } else { - cc.as_os_str().to_owned() - } - }; - - // MSVC with CMake uses msbuild by default which doesn't respect these - // vars that we'd otherwise configure. In that case we just skip this - // entirely. - if target.is_msvc() && !builder.ninja() { - return; - } - - let (cc, cxx) = match builder.config.llvm_clang_cl { - Some(ref cl) => (cl.into(), cl.into()), - None => (builder.cc(target), builder.cxx(target).unwrap()), - }; - - // Handle msvc + ninja + ccache specially (this is what the bots use) - if target.is_msvc() && builder.ninja() && builder.config.ccache.is_some() { - let mut wrap_cc = env::current_exe().expect("failed to get cwd"); - wrap_cc.set_file_name("sccache-plus-cl.exe"); - - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); - cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) - .env("SCCACHE_TARGET", target.triple) - .env("SCCACHE_CC", &cc) - .env("SCCACHE_CXX", &cxx); - - // Building LLVM on MSVC can be a little ludicrous at times. We're so far - // off the beaten path here that I'm not really sure this is even half - // supported any more. Here we're trying to: - // - // * Build LLVM on MSVC - // * Build LLVM with `clang-cl` instead of `cl.exe` - // * Build a project with `sccache` - // * Build for 32-bit as well - // * Build with Ninja - // - // For `cl.exe` there are different binaries to compile 32/64 bit which - // we use but for `clang-cl` there's only one which internally - // multiplexes via flags. As a result it appears that CMake's detection - // of a compiler's architecture and such on MSVC **doesn't** pass any - // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we - // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which - // definitely causes problems since all the env vars are pointing to - // 32-bit libraries. - // - // To hack around this... again... we pass an argument that's - // unconditionally passed in the sccache shim. This'll get CMake to - // correctly diagnose it's doing a 32-bit compilation and LLVM will - // internally configure itself appropriately. - if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { - cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); - } - } else { - // If ccache is configured we inform the build a little differently how - // to invoke ccache while also invoking our compilers. - if use_compiler_launcher { - if let Some(ref ccache) = builder.config.ccache { - cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) - .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); - } - } - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&cxx)) - .define("CMAKE_ASM_COMPILER", sanitize_cc(&cc)); - } - - cfg.build_arg("-j").build_arg(builder.jobs().to_string()); - let mut cflags: OsString = builder - .cflags(target, GitRepo::Llvm, CLang::C) - .into_iter() - .filter(|flag| { - !suppressed_compiler_flag_prefixes - .iter() - .any(|suppressed_prefix| flag.starts_with(suppressed_prefix)) - }) - .collect::>() - .join(" ") - .into(); - if let Some(ref s) = builder.config.llvm_cflags { - cflags.push(" "); - cflags.push(s); - } - - if builder.config.llvm_clang_cl.is_some() { - cflags.push(format!(" --target={target}")); - } - cfg.define("CMAKE_C_FLAGS", cflags); - let mut cxxflags: OsString = builder - .cflags(target, GitRepo::Llvm, CLang::Cxx) - .into_iter() - .filter(|flag| { - !suppressed_compiler_flag_prefixes - .iter() - .any(|suppressed_prefix| flag.starts_with(suppressed_prefix)) - }) - .collect::>() - .join(" ") - .into(); - if let Some(ref s) = builder.config.llvm_cxxflags { - cxxflags.push(" "); - cxxflags.push(s); - } - if builder.config.llvm_clang_cl.is_some() { - cxxflags.push(format!(" --target={target}")); - } - cfg.define("CMAKE_CXX_FLAGS", cxxflags); - if let Some(ar) = builder.ar(target) { - if ar.is_absolute() { - // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_AR", sanitize_cc(&ar)); - } - } - - if let Some(ranlib) = builder.ranlib(target) { - if ranlib.is_absolute() { - // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_RANLIB", sanitize_cc(&ranlib)); - } - } - - if let Some(ref flags) = builder.config.llvm_ldflags { - ldflags.push_all(flags); - } - - if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { - ldflags.push_all(&flags); - } - - // For distribution we want the LLVM tools to be *statically* linked to libstdc++. - // We also do this if the user explicitly requested static libstdc++. - if builder.config.llvm_static_stdcpp - && !target.is_msvc() - && !target.contains("netbsd") - && !target.contains("solaris") - { - if target.contains("apple") || target.is_windows() { - ldflags.push_all("-static-libstdc++"); - } else { - ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); - } - } - - cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); - cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); - cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); - - if env::var_os("SCCACHE_ERROR_LOG").is_some() { - cfg.env("RUSTC_LOG", "sccache=warn"); - } -} - -fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { - // ThinLTO is only available when building with LLVM, enabling LLD is required. - // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. - if builder.config.llvm_thin_lto { - cfg.define("LLVM_ENABLE_LTO", "Thin"); - if !target.contains("apple") { - cfg.define("LLVM_ENABLE_LLD", "ON"); - } - } - - // Libraries for ELF section compression. - if builder.config.llvm_libzstd { - cfg.define("LLVM_ENABLE_ZSTD", "FORCE_ON"); - cfg.define("LLVM_USE_STATIC_ZSTD", "TRUE"); - } else { - cfg.define("LLVM_ENABLE_ZSTD", "OFF"); - } - - if let Some(ref linker) = builder.config.llvm_use_linker { - cfg.define("LLVM_USE_LINKER", linker); - } - - if builder.config.llvm_allow_old_toolchain { - cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); - } -} - -// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 -fn get_var(var_base: &str, host: &str, target: &str) -> Option { - let kind = if host == target { "HOST" } else { "TARGET" }; - let target_u = target.replace('-', "_"); - env::var_os(format!("{var_base}_{target}")) - .or_else(|| env::var_os(format!("{}_{}", var_base, target_u))) - .or_else(|| env::var_os(format!("{}_{}", kind, var_base))) - .or_else(|| env::var_os(var_base)) -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Enzyme { - pub target: TargetSelection, -} - -impl Step for Enzyme { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/enzyme/enzyme") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Enzyme { target: run.target }); - } - - /// Compile Enzyme for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.require_submodule( - "src/tools/enzyme", - Some("The Enzyme sources are required for autodiff."), - ); - if builder.config.dry_run() { - let out_dir = builder.enzyme_out(self.target); - return out_dir; - } - let target = self.target; - - let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: self.target }); - - static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - generate_smart_stamp_hash( - builder, - &builder.config.src.join("src/tools/enzyme"), - builder.enzyme_info.sha().unwrap_or_default(), - ) - }); - - let out_dir = builder.enzyme_out(target); - let stamp = out_dir.join("enzyme-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info( - "Could not determine the Enzyme submodule commit hash. \ - Assuming that an Enzyme rebuild is not necessary.", - ); - builder.info(&format!( - "To force Enzyme to rebuild, remove the file `{}`", - stamp.path.display() - )); - } - return out_dir; - } - - builder.info(&format!("Building Enzyme for {}", target)); - t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - builder - .config - .update_submodule(Path::new("src").join("tools").join("enzyme").to_str().unwrap()); - let mut cfg = cmake::Config::new(builder.src.join("src/tools/enzyme/enzyme/")); - // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds - //cfg.profile("Debug"); - //cfg.define("CMAKE_BUILD_TYPE", "Debug"); - configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); - - // Re-use the same flags as llvm to control the level of debug information - // generated for lld. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - cfg.out_dir(&out_dir) - .profile(profile) - .env("LLVM_CONFIG_REAL", &llvm_config) - .define("LLVM_ENABLE_ASSERTIONS", "ON") - .define("ENZYME_EXTERNAL_SHARED_LIB", "ON") - .define("LLVM_DIR", builder.llvm_out(target)); - - cfg.build(); - - t!(stamp.write()); - out_dir - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Lld { - pub target: TargetSelection, -} - -impl Step for Lld { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/lld") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Lld { target: run.target }); - } - - /// Compile LLD for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::from("lld-out-dir-test-gen"); - } - let target = self.target; - - let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); - - // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path - // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` - // subfolder. We check if that's the case, and if LLD's binary already exists there next to - // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. - let ci_llvm_bin = llvm_config.parent().unwrap(); - if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { - let lld_path = ci_llvm_bin.join(exe("lld", target)); - if lld_path.exists() { - // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the - // `bin` subfolder of this step's out dir. - return ci_llvm_bin.parent().unwrap().to_path_buf(); - } - } - - let out_dir = builder.lld_out(target); - let done_stamp = out_dir.join("lld-finished-building"); - if done_stamp.exists() { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); - let mut ldflags = LdFlags::default(); - - // When building LLD as part of a build with instrumentation on windows, for example - // when doing PGO on CI, cmake or clang-cl don't automatically link clang's - // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid - // linking errors, much like LLVM's cmake setup does in that situation. - if builder.config.llvm_profile_generate && target.is_msvc() { - if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { - // Find clang's runtime library directory and push that as a search path to the - // cmake linker flags. - let clang_rt_dir = get_clang_cl_resource_dir(builder, clang_cl_path); - ldflags.push_all(format!("/libpath:{}", clang_rt_dir.display())); - } - } - - // LLD is built as an LLVM tool, but is distributed outside of the `llvm-tools` component, - // which impacts where it expects to find LLVM's shared library. This causes #80703. - // - // LLD is distributed at "$root/lib/rustlib/$host/bin/rust-lld", but the `libLLVM-*.so` it - // needs is distributed at "$root/lib". The default rpath of "$ORIGIN/../lib" points at the - // lib path for LLVM tools, not the one for rust binaries. - // - // (The `llvm-tools` component copies the .so there for the other tools, and with that - // component installed, one can successfully invoke `rust-lld` directly without rustup's - // `LD_LIBRARY_PATH` overrides) - // - if builder.config.rpath_enabled(target) - && helpers::use_host_linker(target) - && builder.config.llvm_link_shared() - && target.contains("linux") - { - // So we inform LLD where it can find LLVM's libraries by adding an rpath entry to the - // expected parent `lib` directory. - // - // Be careful when changing this path, we need to ensure it's quoted or escaped: - // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to - // cmake. - ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - // Re-use the same flags as llvm to control the level of debug information - // generated for lld. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_CMAKE_DIR", llvm_cmake_dir) - .define("LLVM_INCLUDE_TESTS", "OFF"); - - if target != builder.config.build { - // Use the host llvm-tblgen binary. - cfg.define( - "LLVM_TABLEGEN_EXE", - llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - } - - cfg.build(); - - t!(File::create(&done_stamp)); - out_dir - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Sanitizers { - pub target: TargetSelection, -} - -impl Step for Sanitizers { - type Output = Vec; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("sanitizers") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Sanitizers { target: run.target }); - } - - /// Builds sanitizer runtime libraries. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); - if !compiler_rt_dir.exists() { - return Vec::new(); - } - - let out_dir = builder.native_dir(self.target).join("sanitizers"); - let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); - if runtimes.is_empty() { - return runtimes; - } - - let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if builder.config.dry_run() { - return runtimes; - } - - let stamp = out_dir.join("sanitizers-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info(&format!( - "Rebuild sanitizers by removing the file `{}`", - stamp.path.display() - )); - } - return runtimes; - } - - let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); - t!(stamp.remove()); - let _time = helpers::timeit(builder); - - let mut cfg = cmake::Config::new(&compiler_rt_dir); - cfg.profile("Release"); - cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); - cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); - cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); - cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); - cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); - cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); - cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); - cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); - cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); - cfg.define("LLVM_CONFIG_PATH", &llvm_config); - - // On Darwin targets the sanitizer runtimes are build as universal binaries. - // Unfortunately sccache currently lacks support to build them successfully. - // Disable compiler launcher on Darwin targets to avoid potential issues. - let use_compiler_launcher = !self.target.contains("apple-darwin"); - // Since v1.0.86, the cc crate adds -mmacosx-version-min to the default - // flags on MacOS. A long-standing bug in the CMake rules for compiler-rt - // causes architecture detection to be skipped when this flag is present, - // and compilation fails. https://github.com/llvm/llvm-project/issues/88780 - let suppressed_compiler_flag_prefixes: &[&str] = - if self.target.contains("apple-darwin") { &["-mmacosx-version-min="] } else { &[] }; - configure_cmake( - builder, - self.target, - &mut cfg, - use_compiler_launcher, - LdFlags::default(), - suppressed_compiler_flag_prefixes, - ); - - t!(fs::create_dir_all(&out_dir)); - cfg.out_dir(out_dir); - - for runtime in &runtimes { - cfg.build_target(&runtime.cmake_target); - cfg.build(); - } - t!(stamp.write()); - - runtimes - } -} - -#[derive(Clone, Debug)] -pub struct SanitizerRuntime { - /// CMake target used to build the runtime. - pub cmake_target: String, - /// Path to the built runtime library. - pub path: PathBuf, - /// Library filename that will be used rustc. - pub name: String, -} - -/// Returns sanitizers available on a given target. -fn supported_sanitizers( - out_dir: &Path, - target: TargetSelection, - channel: &str, -) -> Vec { - let darwin_libs = |os: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), - path: out_dir - .join(format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), - name: format!("librustc-{}_rt.{}.dylib", channel, c), - }) - .collect() - }; - - let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}-{}", c, arch), - path: out_dir.join(format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), - name: format!("librustc-{}_rt.{}.a", channel, c), - }) - .collect() - }; - - match &*target.triple { - "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-apple-ios" => darwin_libs("ios", &["asan", "tsan"]), - "aarch64-apple-ios-sim" => darwin_libs("iossim", &["asan", "tsan"]), - "aarch64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), - "aarch64-unknown-linux-gnu" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "aarch64-unknown-linux-ohos" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "loongarch64-unknown-linux-gnu" | "loongarch64-unknown-linux-musl" => { - common_libs("linux", "loongarch64", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), - "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), - "x86_64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), - "x86_64-unknown-netbsd" => { - common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), - "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), - "x86_64-unknown-linux-gnu" => { - common_libs("linux", "x86_64", &["asan", "dfsan", "lsan", "msan", "safestack", "tsan"]) - } - "x86_64-unknown-linux-musl" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-gnu" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-musl" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-linux-ohos" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - _ => Vec::new(), - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrtBeginEnd { - pub target: TargetSelection, -} - -impl Step for CrtBeginEnd { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/compiler-rt/lib/crt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrtBeginEnd { target: run.target }); - } - - /// Build crtbegin.o/crtend.o for musl target. - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.require_submodule( - "src/llvm-project", - Some("The LLVM sources are required for the CRT from `compiler-rt`."), - ); - - let out_dir = builder.native_dir(self.target).join("crt"); - - if builder.config.dry_run() { - return out_dir; - } - - let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); - let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); - if up_to_date(&crtbegin_src, &out_dir.join("crtbeginS.o")) - && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) - { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "crtbegin.o and crtend.o", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cc::Build::new(); - - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(self.target)); - cfg.cargo_metadata(false) - .out_dir(&out_dir) - .target(&self.target.triple) - .host(&builder.config.build.triple) - .warnings(false) - .debug(false) - .opt_level(3) - .file(crtbegin_src) - .file(crtend_src); - - // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt - // Currently only consumer of those objects is musl, which use .init_array/.fini_array - // instead of .ctors/.dtors - cfg.flag("-std=c11") - .define("CRT_HAS_INITFINI_ARRAY", None) - .define("EH_USE_FRAME_REGISTRY", None); - - let objs = cfg.compile_intermediates(); - assert_eq!(objs.len(), 2); - for obj in objs { - let base_name = unhashed_basename(&obj); - assert!(base_name == "crtbegin" || base_name == "crtend"); - t!(fs::copy(&obj, out_dir.join(format!("{}S.o", base_name)))); - t!(fs::rename(&obj, out_dir.join(format!("{}.o", base_name)))); - } - - out_dir - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Libunwind { - pub target: TargetSelection, -} - -impl Step for Libunwind { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/libunwind") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Libunwind { target: run.target }); - } - - /// Build libunwind.a - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.require_submodule( - "src/llvm-project", - Some("The LLVM sources are required for libunwind."), - ); - - if builder.config.dry_run() { - return PathBuf::new(); - } - - let out_dir = builder.native_dir(self.target).join("libunwind"); - let root = builder.src.join("src/llvm-project/libunwind"); - - if up_to_date(&root, &out_dir.join("libunwind.a")) { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "libunwind.a", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cc_cfg = cc::Build::new(); - let mut cpp_cfg = cc::Build::new(); - - cpp_cfg.cpp(true); - cpp_cfg.cpp_set_stdlib(None); - cpp_cfg.flag("-nostdinc++"); - cpp_cfg.flag("-fno-exceptions"); - cpp_cfg.flag("-fno-rtti"); - cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); - - for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.target(&self.target.triple); - cfg.host(&builder.config.build.triple); - cfg.warnings(false); - cfg.debug(false); - // get_compiler() need set opt_level first. - cfg.opt_level(3); - cfg.flag("-fstrict-aliasing"); - cfg.flag("-funwind-tables"); - cfg.flag("-fvisibility=hidden"); - cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); - cfg.include(root.join("include")); - cfg.cargo_metadata(false); - cfg.out_dir(&out_dir); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - cfg.static_flag(true); - cfg.flag("-fno-stack-protector"); - cfg.flag("-ffreestanding"); - cfg.flag("-fexceptions"); - - // easiest way to undefine since no API available in cc::Build to undefine - cfg.flag("-U_FORTIFY_SOURCE"); - cfg.define("_FORTIFY_SOURCE", "0"); - cfg.define("RUST_SGX", "1"); - cfg.define("__NO_STRING_INLINES", None); - cfg.define("__NO_MATH_INLINES", None); - cfg.define("_LIBUNWIND_IS_BAREMETAL", None); - cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); - cfg.define("NDEBUG", None); - } - if self.target.is_windows() { - cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); - cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); - } - } - - cc_cfg.compiler(builder.cc(self.target)); - if let Ok(cxx) = builder.cxx(self.target) { - cpp_cfg.compiler(cxx); - } else { - cc_cfg.compiler(builder.cc(self.target)); - } - - // Don't set this for clang - // By default, Clang builds C code in GNU C17 mode. - // By default, Clang builds C++ code according to the C++98 standard, - // with many C++11 features accepted as extensions. - if cc_cfg.get_compiler().is_like_gnu() { - cc_cfg.flag("-std=c99"); - } - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.flag("-std=c++11"); - } - - if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { - // use the same GCC C compiler command to compile C++ code so we do not need to setup the - // C++ compiler env variables on the builders. - // Don't set this for clang++, as clang++ is able to compile this without libc++. - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.cpp(false); - cpp_cfg.compiler(builder.cc(self.target)); - } - } - - let mut c_sources = vec![ - "Unwind-sjlj.c", - "UnwindLevel1-gcc-ext.c", - "UnwindLevel1.c", - "UnwindRegistersRestore.S", - "UnwindRegistersSave.S", - ]; - - let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; - let cpp_len = cpp_sources.len(); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - c_sources.push("UnwindRustSgx.c"); - } - - for src in c_sources { - cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - for src in &cpp_sources { - cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - cpp_cfg.compile("unwind-cpp"); - - // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 - let mut count = 0; - for entry in fs::read_dir(&out_dir).unwrap() { - let file = entry.unwrap().path().canonicalize().unwrap(); - if file.is_file() && file.extension() == Some(OsStr::new("o")) { - // Object file name without the hash prefix is "Unwind-EHABI", "Unwind-seh" or "libunwind". - let base_name = unhashed_basename(&file); - if cpp_sources.iter().any(|f| *base_name == f[..f.len() - 4]) { - cc_cfg.object(&file); - count += 1; - } - } - } - assert_eq!(cpp_len, count, "Can't get object files from {out_dir:?}"); - - cc_cfg.compile("unwind"); - out_dir - } -} diff --git a/standalonex/src/src/core/build_steps/mod.rs b/standalonex/src/src/core/build_steps/mod.rs deleted file mode 100644 index fcb6abea..00000000 --- a/standalonex/src/src/core/build_steps/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub(crate) mod check; -pub(crate) mod clean; -pub(crate) mod clippy; -pub(crate) mod compile; -pub(crate) mod dist; -pub(crate) mod doc; -pub(crate) mod format; -pub(crate) mod gcc; -pub(crate) mod install; -pub(crate) mod llvm; -pub(crate) mod perf; -pub(crate) mod run; -pub(crate) mod setup; -pub(crate) mod suggest; -pub(crate) mod synthetic_targets; -pub(crate) mod test; -pub(crate) mod tool; -pub(crate) mod toolstate; -pub(crate) mod vendor; diff --git a/standalonex/src/src/core/build_steps/perf.rs b/standalonex/src/src/core/build_steps/perf.rs deleted file mode 100644 index 5b83080a..00000000 --- a/standalonex/src/src/core/build_steps/perf.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::core::build_steps::compile::{Std, Sysroot}; -use crate::core::build_steps::tool::{RustcPerf, Tool}; -use crate::core::builder::Builder; -use crate::core::config::DebuginfoLevel; - -/// Performs profiling using `rustc-perf` on a built version of the compiler. -pub fn perf(builder: &Builder<'_>) { - let collector = builder.ensure(RustcPerf { - compiler: builder.compiler(0, builder.config.build), - target: builder.config.build, - }); - - if builder.build.config.rust_debuginfo_level_rustc == DebuginfoLevel::None { - builder.info(r#"WARNING: You are compiling rustc without debuginfo, this will make profiling less useful. -Consider setting `rust.debuginfo-level = 1` in `config.toml`."#); - } - - let compiler = builder.compiler(builder.top_stage, builder.config.build); - builder.ensure(Std::new(compiler, builder.config.build)); - let sysroot = builder.ensure(Sysroot::new(compiler)); - let rustc = sysroot.join("bin/rustc"); - - let rustc_perf_dir = builder.build.tempdir().join("rustc-perf"); - let profile_results_dir = rustc_perf_dir.join("results"); - - // We need to take args passed after `--` and pass them to `rustc-perf-wrapper` - let args = std::env::args().skip_while(|a| a != "--").skip(1); - - let mut cmd = builder.tool_cmd(Tool::RustcPerfWrapper); - cmd.env("RUSTC_REAL", rustc) - .env("PERF_COLLECTOR", collector) - .env("PERF_RESULT_DIR", profile_results_dir) - .args(args); - cmd.run(builder); -} diff --git a/standalonex/src/src/core/build_steps/run.rs b/standalonex/src/src/core/build_steps/run.rs deleted file mode 100644 index a6dff7fd..00000000 --- a/standalonex/src/src/core/build_steps/run.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Build-and-run steps for in-repo tools -//! -//! A bit of a hodge-podge as e.g. if a tool's a test fixture it should be in `build_steps::test`. -//! If it can be reached from `./x.py run` it can go here. - -use std::path::PathBuf; - -use crate::Mode; -use crate::core::build_steps::dist::distdir; -use crate::core::build_steps::test; -use crate::core::build_steps::tool::{self, SourceType, Tool}; -use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::core::config::TargetSelection; -use crate::core::config::flags::get_completion; -use crate::utils::exec::command; - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct BuildManifest; - -impl Step for BuildManifest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest); - } - - fn run(self, builder: &Builder<'_>) { - // This gets called by `promote-release` - // (https://github.com/rust-lang/promote-release). - let mut cmd = builder.tool_cmd(Tool::BuildManifest); - let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") - }); - let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") - }); - - let today = command("date").arg("+%Y-%m-%d").run_capture_stdout(builder).stdout(); - - cmd.arg(sign); - cmd.arg(distdir(builder)); - cmd.arg(today.trim()); - cmd.arg(addr); - cmd.arg(&builder.config.channel); - - builder.create_dir(&distdir(builder)); - cmd.run(builder); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct BumpStage0; - -impl Step for BumpStage0 { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/bump-stage0") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BumpStage0); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::BumpStage0); - cmd.args(builder.config.args()); - cmd.run(builder); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct ReplaceVersionPlaceholder; - -impl Step for ReplaceVersionPlaceholder { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/replace-version-placeholder") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReplaceVersionPlaceholder); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::ReplaceVersionPlaceholder); - cmd.arg(&builder.src); - cmd.run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - target: TargetSelection, -} - -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let host = builder.build.build; - let target = self.target; - let stage = builder.top_stage; - if stage == 0 { - eprintln!("miri cannot be run at stage 0"); - std::process::exit(1); - } - - // This compiler runs on the host, we'll just use it for the target. - let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); - - // Get a target sysroot for Miri. - let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target); - - // # Run miri. - // Running it via `cargo run` as that figures out the right dylib path. - // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so. - let mut miri = tool::prepare_tool_cargo( - builder, - host_compiler, - Mode::ToolRustc, - host, - Kind::Run, - "src/tools/miri", - SourceType::InTree, - &[], - ); - miri.add_rustc_lib_path(builder); - miri.arg("--").arg("--target").arg(target.rustc_target_arg()); - - // miri tests need to know about the stage sysroot - miri.arg("--sysroot").arg(miri_sysroot); - - // Forward arguments. This may contain further arguments to the program - // after another --, so this must be at the end. - miri.args(builder.config.args()); - - miri.into_cmd().run(builder); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct CollectLicenseMetadata; - -impl Step for CollectLicenseMetadata { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/collect-license-metadata") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CollectLicenseMetadata); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Some(reuse) = &builder.config.reuse else { - panic!("REUSE is required to collect the license metadata"); - }; - - // Temporary location, it will be moved to src/etc once it's accurate. - let dest = builder.out.join("license-metadata.json"); - - let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); - cmd.env("REUSE_EXE", reuse); - cmd.env("DEST", &dest); - cmd.run(builder); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct GenerateCopyright; - -impl Step for GenerateCopyright { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-copyright") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCopyright); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let license_metadata = builder.ensure(CollectLicenseMetadata); - - // Temporary location, it will be moved to the proper one once it's accurate. - let dest = builder.out.join("COPYRIGHT.html"); - - let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); - cmd.env("LICENSE_METADATA", &license_metadata); - cmd.env("DEST", &dest); - cmd.env("OUT_DIR", &builder.out); - cmd.env("CARGO", &builder.initial_cargo); - cmd.run(builder); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct GenerateWindowsSys; - -impl Step for GenerateWindowsSys { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-windows-sys") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateWindowsSys); - } - - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::GenerateWindowsSys); - cmd.arg(&builder.src); - cmd.run(builder); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GenerateCompletions; - -macro_rules! generate_completions { - ( $( ( $shell:ident, $filename:expr ) ),* ) => { - $( - if let Some(comp) = get_completion($shell, &$filename) { - std::fs::write(&$filename, comp).expect(&format!("writing {} completion", stringify!($shell))); - } - )* - }; -} - -impl Step for GenerateCompletions { - type Output = (); - - /// Uses `clap_complete` to generate shell completions. - fn run(self, builder: &Builder<'_>) { - use clap_complete::shells::{Bash, Fish, PowerShell, Zsh}; - - generate_completions!( - (Bash, builder.src.join("src/etc/completions/x.py.sh")), - (Zsh, builder.src.join("src/etc/completions/x.py.zsh")), - (Fish, builder.src.join("src/etc/completions/x.py.fish")), - (PowerShell, builder.src.join("src/etc/completions/x.py.ps1")) - ); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("generate-completions") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCompletions); - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct UnicodeTableGenerator; - -impl Step for UnicodeTableGenerator { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/unicode-table-generator") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnicodeTableGenerator); - } - - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::UnicodeTableGenerator); - cmd.arg(builder.src.join("library/core/src/unicode/unicode_data.rs")); - cmd.run(builder); - } -} diff --git a/standalonex/src/src/core/build_steps/setup.rs b/standalonex/src/src/core/build_steps/setup.rs deleted file mode 100644 index 704fa46a..00000000 --- a/standalonex/src/src/core/build_steps/setup.rs +++ /dev/null @@ -1,740 +0,0 @@ -//! First time setup of a dev environment -//! -//! These are build-and-run steps for `./x.py setup`, which allows quickly setting up the directory -//! for modifying, building, and running the compiler and library. Running arbitrary configuration -//! allows setting up things that cannot be simply captured inside the config.toml, in addition to -//! leading people away from manually editing most of the config.toml values. - -use std::env::consts::EXE_SUFFIX; -use std::fmt::Write as _; -use std::fs::File; -use std::io::Write; -use std::path::{MAIN_SEPARATOR_STR, Path, PathBuf}; -use std::str::FromStr; -use std::{fmt, fs, io}; - -use sha2::Digest; - -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::utils::change_tracker::CONFIG_CHANGE_HISTORY; -use crate::utils::exec::command; -use crate::utils::helpers::{self, hex_encode}; -use crate::{Config, t}; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum Profile { - Compiler, - Library, - Tools, - Dist, - None, -} - -static PROFILE_DIR: &str = "src/bootstrap/defaults"; - -impl Profile { - fn include_path(&self, src_path: &Path) -> PathBuf { - PathBuf::from(format!("{}/{PROFILE_DIR}/config.{}.toml", src_path.display(), self)) - } - - pub fn all() -> impl Iterator { - use Profile::*; - // N.B. these are ordered by how they are displayed, not alphabetically - [Library, Compiler, Tools, Dist, None].iter().copied() - } - - pub fn purpose(&self) -> String { - use Profile::*; - match self { - Library => "Contribute to the standard library", - Compiler => "Contribute to the compiler itself", - Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", - Dist => "Install Rust from source", - None => "Do not modify `config.toml`" - } - .to_string() - } - - pub fn all_for_help(indent: &str) -> String { - let mut out = String::new(); - for choice in Profile::all() { - writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); - } - out - } - - pub fn as_str(&self) -> &'static str { - match self { - Profile::Compiler => "compiler", - Profile::Library => "library", - Profile::Tools => "tools", - Profile::Dist => "dist", - Profile::None => "none", - } - } -} - -impl FromStr for Profile { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "lib" | "library" => Ok(Profile::Library), - "compiler" => Ok(Profile::Compiler), - "maintainer" | "dist" | "user" => Ok(Profile::Dist), - "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { - Ok(Profile::Tools) - } - "none" => Ok(Profile::None), - "llvm" | "codegen" => Err("the \"llvm\" and \"codegen\" profiles have been removed,\ - use \"compiler\" instead which has the same functionality" - .to_string()), - _ => Err(format!("unknown profile: '{s}'")), - } - } -} - -impl fmt::Display for Profile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl Step for Profile { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { - for choice in Profile::all() { - run = run.alias(choice.as_str()); - } - run - } - - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - - let path = &run.builder.config.config.clone().unwrap_or(PathBuf::from("config.toml")); - if path.exists() { - eprintln!(); - eprintln!( - "ERROR: you asked for a new config file, but one already exists at `{}`", - t!(path.canonicalize()).display() - ); - - match prompt_user( - "Do you wish to override the existing configuration (which will allow the setup process to continue)?: [y/N]", - ) { - Ok(Some(PromptResult::Yes)) => { - t!(fs::remove_file(path)); - } - _ => { - println!("Exiting."); - crate::exit!(1); - } - } - } - - // for Profile, `run.paths` will have 1 and only 1 element - // this is because we only accept at most 1 path from user input. - // If user calls `x.py setup` without arguments, the interactive TUI - // will guide user to provide one. - let profile = if run.paths.len() > 1 { - // HACK: `builder` runs this step with all paths if no path was passed. - t!(interactive_path()) - } else { - run.paths - .first() - .unwrap() - .assert_single_path() - .path - .as_path() - .as_os_str() - .to_str() - .unwrap() - .parse() - .unwrap() - }; - - run.builder.ensure(profile); - } - - fn run(self, builder: &Builder<'_>) { - setup(&builder.build.config, self); - } -} - -pub fn setup(config: &Config, profile: Profile) { - let suggestions: &[&str] = match profile { - Profile::Compiler | Profile::None => &["check", "build", "test"], - Profile::Tools => &[ - "check", - "build", - "test tests/rustdoc*", - "test src/tools/clippy", - "test src/tools/miri", - "test src/tools/rustfmt", - ], - Profile::Library => &["check", "build", "test library/std", "doc"], - Profile::Dist => &["dist", "build"], - }; - - println!(); - - println!("To get started, try one of the following commands:"); - for cmd in suggestions { - println!("- `x.py {cmd}`"); - } - - if profile != Profile::Dist { - println!( - "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" - ); - } - - if profile == Profile::Tools { - eprintln!(); - eprintln!( - "NOTE: the `tools` profile sets up the `stage2` toolchain (use \ - `rustup toolchain link 'name' build/host/stage2` to use rustc)" - ) - } - - let path = &config.config.clone().unwrap_or(PathBuf::from("config.toml")); - setup_config_toml(path, profile, config); -} - -fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { - if profile == Profile::None { - return; - } - - let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap().change_id; - let settings = format!( - "# Includes one of the default files in {PROFILE_DIR}\n\ - profile = \"{profile}\"\n\ - change-id = {latest_change_id}\n" - ); - - t!(fs::write(path, settings)); - - let include_path = profile.include_path(&config.src); - println!("`x.py` will now use the configuration at {}", include_path.display()); -} - -/// Creates a toolchain link for stage1 using `rustup` -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct Link; -impl Step for Link { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("link") - } - - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "link" { - run.builder.ensure(Link); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - - if config.dry_run() { - return; - } - - if !rustup_installed(builder) { - println!("WARNING: `rustup` is not installed; Skipping `stage1` toolchain linking."); - return; - } - - let stage_path = - ["build", config.build.rustc_target_arg(), "stage1"].join(MAIN_SEPARATOR_STR); - - if stage_dir_exists(&stage_path[..]) && !config.dry_run() { - attempt_toolchain_link(builder, &stage_path[..]); - } - } -} - -fn rustup_installed(builder: &Builder<'_>) -> bool { - let mut rustup = command("rustup"); - rustup.arg("--version"); - - rustup.allow_failure().run_always().run_capture_stdout(builder).is_success() -} - -fn stage_dir_exists(stage_path: &str) -> bool { - match fs::create_dir(stage_path) { - Ok(_) => true, - Err(_) => Path::new(&stage_path).exists(), - } -} - -fn attempt_toolchain_link(builder: &Builder<'_>, stage_path: &str) { - if toolchain_is_linked(builder) { - return; - } - - if !ensure_stage1_toolchain_placeholder_exists(stage_path) { - eprintln!( - "Failed to create a template for stage 1 toolchain or confirm that it already exists" - ); - return; - } - - if try_link_toolchain(builder, stage_path) { - println!( - "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" - ); - } else { - eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); - eprintln!( - "To manually link stage 1 build to `stage1` toolchain, run:\n - `rustup toolchain link stage1 {}`", - &stage_path - ); - } -} - -fn toolchain_is_linked(builder: &Builder<'_>) -> bool { - match command("rustup") - .allow_failure() - .args(["toolchain", "list"]) - .run_capture_stdout(builder) - .stdout_if_ok() - { - Some(toolchain_list) => { - if !toolchain_list.contains("stage1") { - return false; - } - // The toolchain has already been linked. - println!( - "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" - ); - } - None => { - // In this case, we don't know if the `stage1` toolchain has been linked; - // but `rustup` failed, so let's not go any further. - println!( - "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" - ); - } - } - true -} - -fn try_link_toolchain(builder: &Builder<'_>, stage_path: &str) -> bool { - command("rustup") - .args(["toolchain", "link", "stage1", stage_path]) - .run_capture_stdout(builder) - .is_success() -} - -fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { - let pathbuf = PathBuf::from(stage_path); - - if fs::create_dir_all(pathbuf.join("lib")).is_err() { - return false; - }; - - let pathbuf = pathbuf.join("bin"); - if fs::create_dir_all(&pathbuf).is_err() { - return false; - }; - - let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); - - if pathbuf.exists() { - return true; - } - - // Take care not to overwrite the file - let result = File::options().append(true).create(true).open(&pathbuf); - if result.is_err() { - return false; - } - - true -} - -// Used to get the path for `Subcommand::Setup` -pub fn interactive_path() -> io::Result { - fn abbrev_all() -> impl Iterator { - ('a'..) - .zip(1..) - .map(|(letter, number)| (letter.to_string(), number.to_string())) - .zip(Profile::all()) - } - - fn parse_with_abbrev(input: &str) -> Result { - let input = input.trim().to_lowercase(); - for ((letter, number), profile) in abbrev_all() { - if input == letter || input == number { - return Ok(profile); - } - } - input.parse() - } - - println!("Welcome to the Rust project! What do you want to do with x.py?"); - for ((letter, _), profile) in abbrev_all() { - println!("{}) {}: {}", letter, profile, profile.purpose()); - } - let template = loop { - print!( - "Please choose one ({}): ", - abbrev_all().map(|((l, _), _)| l).collect::>().join("/") - ); - io::stdout().flush()?; - let mut input = String::new(); - io::stdin().read_line(&mut input)?; - if input.is_empty() { - eprintln!("EOF on stdin, when expecting answer to question. Giving up."); - crate::exit!(1); - } - break match parse_with_abbrev(&input) { - Ok(profile) => profile, - Err(err) => { - eprintln!("ERROR: {err}"); - eprintln!("NOTE: press Ctrl+C to exit"); - continue; - } - }; - }; - Ok(template) -} - -#[derive(PartialEq)] -enum PromptResult { - Yes, // y/Y/yes - No, // n/N/no - Print, // p/P/print -} - -/// Prompt a user for a answer, looping until they enter an accepted input or nothing -fn prompt_user(prompt: &str) -> io::Result> { - let mut input = String::new(); - loop { - print!("{prompt} "); - io::stdout().flush()?; - input.clear(); - io::stdin().read_line(&mut input)?; - match input.trim().to_lowercase().as_str() { - "y" | "yes" => return Ok(Some(PromptResult::Yes)), - "n" | "no" => return Ok(Some(PromptResult::No)), - "p" | "print" => return Ok(Some(PromptResult::Print)), - "" => return Ok(None), - _ => { - eprintln!("ERROR: unrecognized option '{}'", input.trim()); - eprintln!("NOTE: press Ctrl+C to exit"); - } - }; - } -} - -/// Installs `src/etc/pre-push.sh` as a Git hook -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct Hook; - -impl Step for Hook { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("hook") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "hook" { - run.builder.ensure(Hook); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - t!(install_git_hook_maybe(builder, config)); - } -} - -// install a git hook to automatically run tidy, if they want -fn install_git_hook_maybe(builder: &Builder<'_>, config: &Config) -> io::Result<()> { - let git = helpers::git(Some(&config.src)) - .args(["rev-parse", "--git-common-dir"]) - .run_capture(builder) - .stdout(); - let git = PathBuf::from(git.trim()); - let hooks_dir = git.join("hooks"); - let dst = hooks_dir.join("pre-push"); - if dst.exists() { - // The git hook has already been set up, or the user already has a custom hook. - return Ok(()); - } - - println!( - "\nRust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. -If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before -pushing your code to ensure your code is up to par. If you decide later that this behavior is -undesirable, simply delete the `pre-push` file from .git/hooks." - ); - - if prompt_user("Would you like to install the git hook?: [y/N]")? != Some(PromptResult::Yes) { - println!("Ok, skipping installation!"); - return Ok(()); - } - if !hooks_dir.exists() { - // We need to (try to) create the hooks directory first. - let _ = fs::create_dir(hooks_dir); - } - let src = config.src.join("src").join("etc").join("pre-push.sh"); - match fs::hard_link(src, &dst) { - Err(e) => { - eprintln!( - "ERROR: could not create hook {}: do you already have the git hook installed?\n{}", - dst.display(), - e - ); - return Err(e); - } - Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), - }; - Ok(()) -} - -/// Handles editor-specific setup differences -#[derive(Clone, Debug, Eq, PartialEq)] -enum EditorKind { - Vscode, - Vim, - Emacs, - Helix, -} - -impl EditorKind { - fn prompt_user() -> io::Result> { - let prompt_str = "Available editors: -1. vscode -2. vim -3. emacs -4. helix - -Select which editor you would like to set up [default: None]: "; - - let mut input = String::new(); - loop { - print!("{}", prompt_str); - io::stdout().flush()?; - input.clear(); - io::stdin().read_line(&mut input)?; - match input.trim().to_lowercase().as_str() { - "1" | "vscode" => return Ok(Some(EditorKind::Vscode)), - "2" | "vim" => return Ok(Some(EditorKind::Vim)), - "3" | "emacs" => return Ok(Some(EditorKind::Emacs)), - "4" | "helix" => return Ok(Some(EditorKind::Helix)), - "" => return Ok(None), - _ => { - eprintln!("ERROR: unrecognized option '{}'", input.trim()); - eprintln!("NOTE: press Ctrl+C to exit"); - } - }; - } - } - - /// A list of historical hashes of each LSP settings file - /// New entries should be appended whenever this is updated so we can detect - /// outdated vs. user-modified settings files. - fn hashes(&self) -> Vec<&str> { - match self { - EditorKind::Vscode | EditorKind::Vim => vec![ - "ea67e259dedf60d4429b6c349a564ffcd1563cf41c920a856d1f5b16b4701ac8", - "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", - "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", - "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", - "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", - "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", - "828666b021d837a33e78d870b56d34c88a5e2c85de58b693607ec574f0c27000", - "811fb3b063c739d261fd8590dd30242e117908f5a095d594fa04585daa18ec4d", - "4eecb58a2168b252077369da446c30ed0e658301efe69691979d1ef0443928f4", - ], - EditorKind::Emacs => vec![ - "51068d4747a13732440d1a8b8f432603badb1864fa431d83d0fd4f8fa57039e0", - "d29af4d949bbe2371eac928a3c31cf9496b1701aa1c45f11cd6c759865ad5c45", - ], - EditorKind::Helix => { - vec!["2d3069b8cf1b977e5d4023965eb6199597755e6c96c185ed5f2854f98b83d233"] - } - } - } - - fn settings_path(&self, config: &Config) -> PathBuf { - config.src.join(self.settings_short_path()) - } - - fn settings_short_path(&self) -> PathBuf { - self.settings_folder().join(match self { - EditorKind::Vscode => "settings.json", - EditorKind::Vim => "coc-settings.json", - EditorKind::Emacs => ".dir-locals.el", - EditorKind::Helix => "languages.toml", - }) - } - - fn settings_folder(&self) -> PathBuf { - match self { - EditorKind::Vscode => PathBuf::from(".vscode"), - EditorKind::Vim => PathBuf::from(".vim"), - EditorKind::Emacs => PathBuf::new(), - EditorKind::Helix => PathBuf::from(".helix"), - } - } - - fn settings_template(&self) -> &str { - match self { - EditorKind::Vscode | EditorKind::Vim => { - include_str!("../../../../etc/rust_analyzer_settings.json") - } - EditorKind::Emacs => include_str!("../../../../etc/rust_analyzer_eglot.el"), - EditorKind::Helix => include_str!("../../../../etc/rust_analyzer_helix.toml"), - } - } - - fn backup_extension(&self) -> String { - format!("{}.bak", self.settings_short_path().extension().unwrap().to_str().unwrap()) - } -} - -/// Sets up or displays the LSP config for one of the supported editors -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct Editor; - -impl Step for Editor { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("editor") - } - - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "editor" { - run.builder.ensure(Editor); - } - } - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - match EditorKind::prompt_user() { - Ok(editor_kind) => { - if let Some(editor_kind) = editor_kind { - while !t!(create_editor_settings_maybe(config, editor_kind.clone())) {} - } else { - println!("Ok, skipping editor setup!"); - } - } - Err(e) => eprintln!("Could not determine the editor: {e}"), - } - } -} - -/// Create the recommended editor LSP config file for rustc development, or just print it -/// If this method should be re-called, it returns `false`. -fn create_editor_settings_maybe(config: &Config, editor: EditorKind) -> io::Result { - let hashes = editor.hashes(); - let (current_hash, historical_hashes) = hashes.split_last().unwrap(); - let settings_path = editor.settings_path(config); - let settings_short_path = editor.settings_short_path(); - let settings_filename = settings_short_path.to_str().unwrap(); - // If None, no settings file exists - // If Some(true), is a previous version of settings.json - // If Some(false), is not a previous version (i.e. user modified) - // If it's up to date we can just skip this - let mut mismatched_settings = None; - if let Ok(current) = fs::read_to_string(&settings_path) { - let mut hasher = sha2::Sha256::new(); - hasher.update(¤t); - let hash = hex_encode(hasher.finalize().as_slice()); - if hash == *current_hash { - return Ok(true); - } else if historical_hashes.contains(&hash.as_str()) { - mismatched_settings = Some(true); - } else { - mismatched_settings = Some(false); - } - } - println!( - "\nx.py can automatically install the recommended `{settings_filename}` file for rustc development" - ); - - match mismatched_settings { - Some(true) => { - eprintln!("WARNING: existing `{settings_filename}` is out of date, x.py will update it") - } - Some(false) => eprintln!( - "WARNING: existing `{settings_filename}` has been modified by user, x.py will back it up and replace it" - ), - _ => (), - } - let should_create = match prompt_user(&format!( - "Would you like to create/update `{settings_filename}`? (Press 'p' to preview values): [y/N]" - ))? { - Some(PromptResult::Yes) => true, - Some(PromptResult::Print) => false, - _ => { - println!("Ok, skipping settings!"); - return Ok(true); - } - }; - if should_create { - let settings_folder_path = config.src.join(editor.settings_folder()); - if !settings_folder_path.exists() { - fs::create_dir(settings_folder_path)?; - } - let verb = match mismatched_settings { - // exists but outdated, we can replace this - Some(true) => "Updated", - // exists but user modified, back it up - Some(false) => { - // exists and is not current version or outdated, so back it up - let backup = settings_path.clone().with_extension(editor.backup_extension()); - eprintln!( - "WARNING: copying `{}` to `{}`", - settings_path.file_name().unwrap().to_str().unwrap(), - backup.file_name().unwrap().to_str().unwrap(), - ); - fs::copy(&settings_path, &backup)?; - "Updated" - } - _ => "Created", - }; - fs::write(&settings_path, editor.settings_template())?; - println!("{verb} `{}`", settings_filename); - } else { - println!("\n{}", editor.settings_template()); - } - Ok(should_create) -} diff --git a/standalonex/src/src/core/build_steps/setup/tests.rs b/standalonex/src/src/core/build_steps/setup/tests.rs deleted file mode 100644 index f3d4b6aa..00000000 --- a/standalonex/src/src/core/build_steps/setup/tests.rs +++ /dev/null @@ -1,17 +0,0 @@ -use sha2::Digest; - -use super::EditorKind; -use crate::utils::helpers::hex_encode; - -#[test] -fn check_matching_settings_hash() { - let editor = EditorKind::Vscode; - let mut hasher = sha2::Sha256::new(); - hasher.update(&editor.settings_template()); - let hash = hex_encode(hasher.finalize().as_slice()); - assert_eq!( - &hash, - editor.hashes().last().unwrap(), - "Update `EditorKind::hashes()` with the new hash of `src/etc/rust_analyzer_settings.json`" - ); -} diff --git a/standalonex/src/src/core/build_steps/suggest.rs b/standalonex/src/src/core/build_steps/suggest.rs deleted file mode 100644 index ba9b1b2f..00000000 --- a/standalonex/src/src/core/build_steps/suggest.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Attempt to magically identify good tests to run - -#![cfg_attr(feature = "build-metrics", allow(unused))] - -use std::path::PathBuf; -use std::str::FromStr; - -use clap::Parser; - -use crate::core::build_steps::tool::Tool; -use crate::core::builder::Builder; - -/// Suggests a list of possible `x.py` commands to run based on modified files in branch. -pub fn suggest(builder: &Builder<'_>, run: bool) { - let git_config = builder.config.git_config(); - let suggestions = builder - .tool_cmd(Tool::SuggestTests) - .env("SUGGEST_TESTS_GIT_REPOSITORY", git_config.git_repository) - .env("SUGGEST_TESTS_NIGHTLY_BRANCH", git_config.nightly_branch) - .env("SUGGEST_TESTS_MERGE_COMMIT_EMAIL", git_config.git_merge_commit_email) - .run_capture_stdout(builder) - .stdout(); - - let suggestions = suggestions - .lines() - .map(|line| { - let mut sections = line.split_ascii_whitespace(); - - // this code expects one suggestion per line in the following format: - // {some number of flags} [optional stage number] - let cmd = sections.next().unwrap(); - let stage = sections.next_back().and_then(|s| str::parse(s).ok()); - let paths: Vec = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); - - (cmd, stage, paths) - }) - .collect::>(); - - if !suggestions.is_empty() { - println!("==== SUGGESTIONS ===="); - for sug in &suggestions { - print!("x {} ", sug.0); - if let Some(stage) = sug.1 { - print!("--stage {stage} "); - } - - for path in &sug.2 { - print!("{} ", path.display()); - } - println!(); - } - println!("====================="); - } else { - println!("No suggestions found!"); - return; - } - - if run { - for sug in suggestions { - let mut build: crate::Build = builder.build.clone(); - build.config.paths = sug.2; - build.config.cmd = crate::core::config::flags::Flags::parse_from(["x.py", sug.0]).cmd; - if let Some(stage) = sug.1 { - build.config.stage = stage; - } - build.build(); - } - } else { - println!("HELP: consider using the `--run` flag to automatically run suggested tests"); - } -} diff --git a/standalonex/src/src/core/build_steps/synthetic_targets.rs b/standalonex/src/src/core/build_steps/synthetic_targets.rs deleted file mode 100644 index 477ff955..00000000 --- a/standalonex/src/src/core/build_steps/synthetic_targets.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! In some cases, parts of bootstrap need to change part of a target spec just for one or a few -//! steps. Adding these targets to rustc proper would "leak" this implementation detail of -//! bootstrap, and would make it more complex to apply additional changes if the need arises. -//! -//! To address that problem, this module implements support for "synthetic targets". Synthetic -//! targets are custom target specs generated using builtin target specs as their base. You can use -//! one of the target specs already defined in this module, or create new ones by adding a new step -//! that calls create_synthetic_target. - -use crate::Compiler; -use crate::core::builder::{Builder, ShouldRun, Step}; -use crate::core::config::TargetSelection; -use crate::utils::exec::command; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct MirOptPanicAbortSyntheticTarget { - pub(crate) compiler: Compiler, - pub(crate) base: TargetSelection, -} - -impl Step for MirOptPanicAbortSyntheticTarget { - type Output = TargetSelection; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - create_synthetic_target(builder, self.compiler, "miropt-abort", self.base, |spec| { - spec.insert("panic-strategy".into(), "abort".into()); - }) - } -} - -fn create_synthetic_target( - builder: &Builder<'_>, - compiler: Compiler, - suffix: &str, - base: TargetSelection, - customize: impl FnOnce(&mut serde_json::Map), -) -> TargetSelection { - if base.contains("synthetic") { - // This check is not strictly needed, but nothing currently needs recursive synthetic - // targets. If the need arises, removing this in the future *SHOULD* be safe. - panic!("cannot create synthetic targets with other synthetic targets as their base"); - } - - let name = format!("{base}-synthetic-{suffix}"); - let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - - if builder.config.dry_run() { - std::fs::write(&path, b"dry run\n").unwrap(); - return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - } - - let mut cmd = command(builder.rustc(compiler)); - cmd.arg("--target").arg(base.rustc_target_arg()); - cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); - - // If `rust.channel` is set to either beta or stable, rustc will complain that - // we cannot use nightly features. So `RUSTC_BOOTSTRAP` is needed here. - cmd.env("RUSTC_BOOTSTRAP", "1"); - - let output = cmd.run_capture(builder).stdout(); - let mut spec: serde_json::Value = serde_json::from_slice(output.as_bytes()).unwrap(); - let spec_map = spec.as_object_mut().unwrap(); - - // The `is-builtin` attribute of a spec needs to be removed, otherwise rustc will complain. - spec_map.remove("is-builtin"); - - customize(spec_map); - - std::fs::write(&path, serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); - TargetSelection::create_synthetic(&name, path.to_str().unwrap()) -} diff --git a/standalonex/src/src/core/build_steps/tool.rs b/standalonex/src/src/core/build_steps/tool.rs deleted file mode 100644 index 3cfbef27..00000000 --- a/standalonex/src/src/core/build_steps/tool.rs +++ /dev/null @@ -1,1154 +0,0 @@ -use std::path::PathBuf; -use std::{env, fs}; - -use crate::core::build_steps::compile; -use crate::core::build_steps::toolstate::ToolState; -use crate::core::builder; -use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::core::config::TargetSelection; -use crate::utils::channel::GitInfo; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::helpers::{add_dylib_path, exe, t}; -use crate::{Compiler, Kind, Mode, gha}; - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub enum SourceType { - InTree, - Submodule, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct ToolBuild { - compiler: Compiler, - target: TargetSelection, - tool: &'static str, - path: &'static str, - mode: Mode, - source_type: SourceType, - extra_features: Vec, - /// Nightly-only features that are allowed (comma-separated list). - allow_features: &'static str, - /// Additional arguments to pass to the `cargo` invocation. - cargo_args: Vec, -} - -impl Builder<'_> { - #[track_caller] - pub(crate) fn msg_tool( - &self, - kind: Kind, - mode: Mode, - tool: &str, - build_stage: u32, - host: &TargetSelection, - target: &TargetSelection, - ) -> Option { - match mode { - // depends on compiler stage, different to host compiler - Mode::ToolRustc => self.msg_sysroot_tool( - kind, - build_stage, - format_args!("tool {tool}"), - *host, - *target, - ), - // doesn't depend on compiler, same as host compiler - _ => self.msg(Kind::Build, build_stage, format_args!("tool {tool}"), *host, *target), - } - } -} - -impl Step for ToolBuild { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Builds a tool in `src/tools` - /// - /// This will build the specified tool with the specified `host` compiler in - /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> PathBuf { - let compiler = self.compiler; - let target = self.target; - let mut tool = self.tool; - let path = self.path; - - match self.mode { - Mode::ToolRustc => { - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Rustc::new(compiler, target)); - } - Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), - Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs - _ => panic!("unexpected Mode for tool build"), - } - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - self.mode, - target, - Kind::Build, - path, - self.source_type, - &self.extra_features, - ); - if !self.allow_features.is_empty() { - cargo.allow_features(self.allow_features); - } - cargo.args(self.cargo_args); - let _guard = builder.msg_tool( - Kind::Build, - self.mode, - self.tool, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - // we check this below - let build_success = compile::stream_cargo(builder, cargo, vec![], &mut |_| {}); - - builder.save_toolstate( - tool, - if build_success { ToolState::TestFail } else { ToolState::BuildFail }, - ); - - if !build_success { - crate::exit!(1); - } else { - // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and - // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something - // different so the problem doesn't come up. - if tool == "tidy" { - tool = "rust-tidy"; - } - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) - } - } -} - -#[allow(clippy::too_many_arguments)] // FIXME: reduce the number of args and remove this. -pub fn prepare_tool_cargo( - builder: &Builder<'_>, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - cmd_kind: Kind, - path: &str, - source_type: SourceType, - extra_features: &[String], -) -> CargoCommand { - let mut cargo = builder::Cargo::new(builder, compiler, mode, source_type, target, cmd_kind); - - let dir = builder.src.join(path); - cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); - - let mut features = extra_features.to_vec(); - if builder.build.config.cargo_native_static { - if path.ends_with("cargo") - || path.ends_with("rls") - || path.ends_with("clippy") - || path.ends_with("miri") - || path.ends_with("rustfmt") - { - cargo.env("LIBZ_SYS_STATIC", "1"); - } - if path.ends_with("cargo") { - features.push("all-static".to_string()); - } - } - - // clippy tests need to know about the stage sysroot. Set them consistently while building to - // avoid rebuilding when running tests. - cargo.env("SYSROOT", builder.sysroot(compiler)); - - // if tools are using lzma we want to force the build script to build its - // own copy - cargo.env("LZMA_API_STATIC", "1"); - - // CFG_RELEASE is needed by rustfmt (and possibly other tools) which - // import rustc-ap-rustc_attr which requires this to be set for the - // `#[cfg(version(...))]` attribute. - cargo.env("CFG_RELEASE", builder.rust_release()); - cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); - cargo.env("CFG_VERSION", builder.rust_version()); - cargo.env("CFG_RELEASE_NUM", &builder.version); - cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - - let info = GitInfo::new(builder.config.omit_git_hash, &dir); - if let Some(sha) = info.sha() { - cargo.env("CFG_COMMIT_HASH", sha); - } - if let Some(sha_short) = info.sha_short() { - cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); - } - if let Some(date) = info.commit_date() { - cargo.env("CFG_COMMIT_DATE", date); - } - if !features.is_empty() { - cargo.arg("--features").arg(features.join(", ")); - } - - // Enable internal lints for clippy and rustdoc - // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` - // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 - // - // NOTE: We unconditionally set this here to avoid recompiling tools between `x check $tool` - // and `x test $tool` executions. - // See https://github.com/rust-lang/rust/issues/116538 - cargo.rustflag("-Zunstable-options"); - - // NOTE: The root cause of needing `-Zon-broken-pipe=kill` in the first place is because `rustc` - // and `rustdoc` doesn't gracefully handle I/O errors due to usages of raw std `println!` macros - // which panics upon encountering broken pipes. `-Zon-broken-pipe=kill` just papers over that - // and stops rustc/rustdoc ICEing on e.g. `rustc --print=sysroot | false`. - // - // cargo explicitly does not want the `-Zon-broken-pipe=kill` paper because it does actually use - // variants of `println!` that handles I/O errors gracefully. It's also a breaking change for a - // spawn process not written in Rust, especially if the language default handler is not - // `SIG_IGN`. Thankfully cargo tests will break if we do set the flag. - // - // For the cargo discussion, see - // . - // - // For the rustc discussion, see - // - // for proper solutions. - if !path.ends_with("cargo") { - // Use an untracked env var `FORCE_ON_BROKEN_PIPE_KILL` here instead of `RUSTFLAGS`. - // `RUSTFLAGS` is tracked by cargo. Conditionally omitting `-Zon-broken-pipe=kill` from - // `RUSTFLAGS` causes unnecessary tool rebuilds due to cache invalidation from building e.g. - // cargo *without* `-Zon-broken-pipe=kill` but then rustdoc *with* `-Zon-broken-pipe=kill`. - cargo.env("FORCE_ON_BROKEN_PIPE_KILL", "-Zon-broken-pipe=kill"); - } - - cargo -} - -/// Links a built tool binary with the given `name` from the build directory to the -/// tools directory. -fn copy_link_tool_bin( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - mode: Mode, - name: &str, -) -> PathBuf { - let cargo_out = builder.cargo_out(compiler, mode, target).join(exe(name, target)); - let bin = builder.tools_dir(compiler).join(exe(name, target)); - builder.copy_link(&cargo_out, &bin); - bin -} - -macro_rules! bootstrap_tool { - ($( - $name:ident, $path:expr, $tool_name:expr - $(,is_external_tool = $external:expr)* - $(,is_unstable_tool = $unstable:expr)* - $(,allow_features = $allow_features:expr)? - $(,submodules = $submodules:expr)? - ; - )+) => { - #[derive(PartialEq, Eq, Clone)] - #[allow(dead_code)] - pub enum Tool { - $( - $name, - )+ - } - - impl<'a> Builder<'a> { - pub fn tool_exe(&self, tool: Tool) -> PathBuf { - match tool { - $(Tool::$name => - self.ensure($name { - compiler: self.compiler(0, self.config.build), - target: self.config.build, - }), - )+ - } - } - } - - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - // snapshot compiler - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - $( - for submodule in $submodules { - builder.require_submodule(submodule, None); - } - )* - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: $tool_name, - mode: if false $(|| $unstable)* { - // use in-tree libraries for unstable features - Mode::ToolStd - } else { - Mode::ToolBootstrap - }, - path: $path, - source_type: if false $(|| $external)* { - SourceType::Submodule - } else { - SourceType::InTree - }, - extra_features: vec![], - allow_features: concat!($($allow_features)*), - cargo_args: vec![] - }) - } - } - )+ - } -} - -bootstrap_tool!( - Rustbook, "src/tools/rustbook", "rustbook", submodules = SUBMODULES_FOR_RUSTBOOK; - UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; - Tidy, "src/tools/tidy", "tidy"; - Linkchecker, "src/tools/linkchecker", "linkchecker"; - CargoTest, "src/tools/cargotest", "cargotest"; - Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; - BuildManifest, "src/tools/build-manifest", "build-manifest"; - RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; - RustInstaller, "src/tools/rust-installer", "rust-installer"; - RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; - LintDocs, "src/tools/lint-docs", "lint-docs"; - JsonDocCk, "src/tools/jsondocck", "jsondocck"; - JsonDocLint, "src/tools/jsondoclint", "jsondoclint"; - HtmlChecker, "src/tools/html-checker", "html-checker"; - BumpStage0, "src/tools/bump-stage0", "bump-stage0"; - ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; - CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; - GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; - SuggestTests, "src/tools/suggest-tests", "suggest-tests"; - GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; - RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; - CoverageDump, "src/tools/coverage-dump", "coverage-dump"; - RustcPerfWrapper, "src/tools/rustc-perf-wrapper", "rustc-perf-wrapper"; - WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld", is_unstable_tool = true, allow_features = "min_specialization"; - UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; -); - -/// These are the submodules that are required for rustbook to work due to -/// depending on mdbook plugins. -pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book", "src/doc/reference"]; - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct OptimizedDist { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for OptimizedDist { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/opt-dist") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(OptimizedDist { - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - // We need to ensure the rustc-perf submodule is initialized when building opt-dist since - // the tool requires it to be in place to run. - builder.require_submodule("src/tools/rustc-perf", None); - - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - -/// The [rustc-perf](https://github.com/rust-lang/rustc-perf) benchmark suite, which is added -/// as a submodule at `src/tools/rustc-perf`. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustcPerf { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustcPerf { - /// Path to the built `collector` binary. - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustc-perf") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcPerf { - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - // We need to ensure the rustc-perf submodule is initialized. - builder.require_submodule("src/tools/rustc-perf", None); - - let tool = ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "collector", - mode: Mode::ToolBootstrap, - path: "src/tools/rustc-perf", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - // Only build the collector package, which is used for benchmarking through - // a CLI. - cargo_args: vec!["-p".to_string(), "collector".to_string()], - }; - let collector_bin = builder.ensure(tool.clone()); - // We also need to symlink the `rustc-fake` binary to the corresponding directory, - // because `collector` expects it in the same directory. - copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); - - collector_bin - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct ErrorIndex { - pub compiler: Compiler, -} - -impl ErrorIndex { - pub fn command(builder: &Builder<'_>) -> BootstrapCommand { - // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` - // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. - let host = builder.config.build; - let compiler = builder.compiler_for(builder.top_stage, host, host); - let mut cmd = command(builder.ensure(ErrorIndex { compiler })); - let mut dylib_paths = builder.rustc_lib_paths(compiler); - dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host))); - add_dylib_path(dylib_paths, &mut cmd); - cmd - } -} - -impl Step for ErrorIndex { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // Compile the error-index in the same stage as rustdoc to avoid - // recompiling rustdoc twice if we can. - // - // NOTE: This `make_run` isn't used in normal situations, only if you - // manually build the tool with `x.py build - // src/tools/error-index-generator` which almost nobody does. - // Normally, `x.py test` or `x.py doc` will use the - // `ErrorIndex::command` function instead. - let compiler = - run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RemoteTestServer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RemoteTestServer { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/remote-test-server") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RemoteTestServer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct Rustdoc { - /// This should only ever be 0 or 2. - /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. - pub compiler: Compiler, -} - -impl Step for Rustdoc { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc").path("src/librustdoc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustdoc { - // NOTE: this is somewhat unique in that we actually want a *target* - // compiler here, because rustdoc *is* a compiler. We won't be using - // this as the compiler to build with, but rather this is "what - // compiler are we producing"? - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let target_compiler = self.compiler; - if target_compiler.stage == 0 { - if !target_compiler.is_snapshot(builder) { - panic!("rustdoc in stage 0 must be snapshot rustdoc"); - } - return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); - } - let target = target_compiler.host; - - let bin_rustdoc = || { - let sysroot = builder.sysroot(target_compiler); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); - let _ = fs::remove_file(&bin_rustdoc); - bin_rustdoc - }; - - // If CI rustc is enabled and we haven't modified the rustdoc sources, - // use the precompiled rustdoc from CI rustc's sysroot to speed up bootstrapping. - if builder.download_rustc() - && target_compiler.stage > 0 - && builder.rust_info().is_managed_git_subrepository() - { - let files_to_track = &["src/librustdoc", "src/tools/rustdoc"]; - - // Check if unchanged - if builder.config.last_modified_commit(files_to_track, "download-rustc", true).is_some() - { - let precompiled_rustdoc = builder - .config - .ci_rustc_dir() - .join("bin") - .join(exe("rustdoc", target_compiler.host)); - - let bin_rustdoc = bin_rustdoc(); - builder.copy_link(&precompiled_rustdoc, &bin_rustdoc); - return bin_rustdoc; - } - } - - let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 { - // We already have the stage 1 compiler, we don't need to cut the stage. - builder.compiler(target_compiler.stage, builder.config.build) - } else { - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - builder.compiler(target_compiler.stage - 1, builder.config.build) - }; - - // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually - // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build - // it. - builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); - builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); - - // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, - // compiler libraries, ...) are built. Rustdoc does not require the presence of any - // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since - // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional - // libraries here. The intuition here is that If we've built a compiler, we should be able - // to build rustdoc. - // - let mut features = Vec::new(); - if builder.config.jemalloc { - features.push("jemalloc".to_string()); - } - - // NOTE: Never modify the rustflags here, it breaks the build cache for other tools! - let cargo = prepare_tool_cargo( - builder, - build_compiler, - Mode::ToolRustc, - target, - Kind::Build, - "src/tools/rustdoc", - SourceType::InTree, - features.as_slice(), - ); - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - "rustdoc", - build_compiler.stage, - &self.compiler.host, - &target, - ); - cargo.into_cmd().run(builder); - - // Cargo adds a number of paths to the dylib search path on windows, which results in - // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" - // rustdoc a different name. - let tool_rustdoc = builder - .cargo_out(build_compiler, Mode::ToolRustc, target) - .join(exe("rustdoc_tool_binary", target_compiler.host)); - - // don't create a stage0-sysroot/bin directory. - if target_compiler.stage > 0 { - let bin_rustdoc = bin_rustdoc(); - builder.copy_link(&tool_rustdoc, &bin_rustdoc); - bin_rustdoc - } else { - tool_rustdoc - } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/cargo").default_condition(builder.tool_enabled("cargo")) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.build.require_submodule("src/tools/cargo", None); - - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct LldWrapper { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LldWrapper { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl RustAnalyzer { - pub const ALLOW_FEATURES: &'static str = "rustc_private,proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink,proc_macro_def_site"; -} - -impl Step for RustAnalyzer { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition(builder.tool_enabled("rust-analyzer")) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzerProcMacroSrv { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzerProcMacroSrv { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. - run.path("src/tools/rust-analyzer") - .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") - .default_condition( - builder.tool_enabled("rust-analyzer") - || builder.tool_enabled("rust-analyzer-proc-macro-srv"), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzerProcMacroSrv { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let path = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }); - - // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` - // so that r-a can use it. - let libexec_path = builder.sysroot(self.compiler).join("libexec"); - t!(fs::create_dir_all(&libexec_path)); - builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); - - Some(path) - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct LlvmBitcodeLinker { - pub compiler: Compiler, - pub target: TargetSelection, - pub extra_features: Vec, -} - -impl Step for LlvmBitcodeLinker { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/llvm-bitcode-linker") - .default_condition(builder.tool_enabled("llvm-bitcode-linker")) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LlvmBitcodeLinker { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - extra_features: Vec::new(), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let bin_name = "llvm-bitcode-linker"; - - // If enabled, use ci-rustc and skip building the in-tree compiler. - if !builder.download_rustc() { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - builder.ensure(compile::Rustc::new(self.compiler, self.target)); - } - - let cargo = prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolRustc, - self.target, - Kind::Build, - "src/tools/llvm-bitcode-linker", - SourceType::InTree, - &self.extra_features, - ); - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - bin_name, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - cargo.into_cmd().run(builder); - - let tool_out = builder - .cargo_out(self.compiler, Mode::ToolRustc, self.target) - .join(exe(bin_name, self.compiler.host)); - - if self.compiler.stage > 0 { - let bindir_self_contained = builder - .sysroot(self.compiler) - .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); - t!(fs::create_dir_all(&bindir_self_contained)); - let bin_destination = bindir_self_contained.join(exe(bin_name, self.compiler.host)); - builder.copy_link(&tool_out, &bin_destination); - bin_destination - } else { - tool_out - } - } -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct LibcxxVersionTool { - pub target: TargetSelection, -} - -#[allow(dead_code)] -#[derive(Debug, Clone)] -pub enum LibcxxVersion { - Gnu(usize), - Llvm(usize), -} - -impl Step for LibcxxVersionTool { - type Output = LibcxxVersion; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> LibcxxVersion { - let out_dir = builder.out.join(self.target.to_string()).join("libcxx-version"); - let executable = out_dir.join(exe("libcxx-version", self.target)); - - // This is a sanity-check specific step, which means it is frequently called (when using - // CI LLVM), and compiling `src/tools/libcxx-version/main.cpp` at the beginning of the bootstrap - // invocation adds a fair amount of overhead to the process (see https://github.com/rust-lang/rust/issues/126423). - // Therefore, we want to avoid recompiling this file unnecessarily. - if !executable.exists() { - if !out_dir.exists() { - t!(fs::create_dir_all(&out_dir)); - } - - let compiler = builder.cxx(self.target).unwrap(); - let mut cmd = command(compiler); - - cmd.arg("-o") - .arg(&executable) - .arg(builder.src.join("src/tools/libcxx-version/main.cpp")); - - cmd.run(builder); - - if !executable.exists() { - panic!("Something went wrong. {} is not present", executable.display()); - } - } - - let version_output = command(executable).run_capture_stdout(builder).stdout(); - - let version_str = version_output.split_once("version:").unwrap().1; - let version = version_str.trim().parse::().unwrap(); - - if version_output.starts_with("libstdc++") { - LibcxxVersion::Gnu(version) - } else if version_output.starts_with("libc++") { - LibcxxVersion::Llvm(version) - } else { - panic!("Coudln't recognize the standard library version."); - } - } -} - -macro_rules! tool_extended { - (($sel:ident, $builder:ident), - $($name:ident, - $path:expr, - $tool_name:expr, - stable = $stable:expr - $(,tool_std = $tool_std:literal)? - $(,allow_features = $allow_features:expr)? - $(,add_bins_to_sysroot = $add_bins_to_sysroot:expr)? - ;)+) => { - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - pub extra_features: Vec, - } - - impl Step for $name { - type Output = PathBuf; - const DEFAULT: bool = true; // Overwritten below - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or( - // By default, on nightly/dev enable all tools, else only - // build stable tools. - $stable || builder.build.unstable_features(), - // If `tools` is set, search list for this tool. - |tools| { - tools.iter().any(|tool| match tool.as_ref() { - "clippy" => $tool_name == "clippy-driver", - x => $tool_name == x, - }) - }), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - extra_features: Vec::new(), - }); - } - - #[allow(unused_mut)] - fn run(mut $sel, $builder: &Builder<'_>) -> PathBuf { - let tool = $builder.ensure(ToolBuild { - compiler: $sel.compiler, - target: $sel.target, - tool: $tool_name, - mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, - path: $path, - extra_features: $sel.extra_features, - source_type: SourceType::InTree, - allow_features: concat!($($allow_features)*), - cargo_args: vec![] - }); - - if (false $(|| !$add_bins_to_sysroot.is_empty())?) && $sel.compiler.stage > 0 { - let bindir = $builder.sysroot($sel.compiler).join("bin"); - t!(fs::create_dir_all(&bindir)); - - #[allow(unused_variables)] - let tools_out = $builder - .cargo_out($sel.compiler, Mode::ToolRustc, $sel.target); - - $(for add_bin in $add_bins_to_sysroot { - let bin_source = tools_out.join(exe(add_bin, $sel.target)); - let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); - $builder.copy_link(&bin_source, &bin_destination); - })? - - let tool = bindir.join(exe($tool_name, $sel.compiler.host)); - tool - } else { - tool - } - } - } - )+ - } -} - -tool_extended!((self, builder), - Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; - CargoClippy, "src/tools/clippy", "cargo-clippy", stable=true; - Clippy, "src/tools/clippy", "clippy-driver", stable=true, add_bins_to_sysroot = ["clippy-driver", "cargo-clippy"]; - Miri, "src/tools/miri", "miri", stable=false, add_bins_to_sysroot = ["miri"]; - CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", stable=false, add_bins_to_sysroot = ["cargo-miri"]; - Rls, "src/tools/rls", "rls", stable=true; - Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; -); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TestFloatParse { - pub host: TargetSelection, -} - -impl Step for TestFloatParse { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/etc/test-float-parse") - } - - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(builder.top_stage, bootstrap_host); - - builder.ensure(ToolBuild { - compiler, - target: bootstrap_host, - tool: "test-float-parse", - mode: Mode::ToolStd, - path: "src/etc/test-float-parse", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }); - } -} - -impl Builder<'_> { - /// Gets a `BootstrapCommand` which is ready to run `tool` in `stage` built for - /// `host`. - pub fn tool_cmd(&self, tool: Tool) -> BootstrapCommand { - let mut cmd = command(self.tool_exe(tool)); - let compiler = self.compiler(0, self.config.build); - let host = &compiler.host; - // Prepares the `cmd` provided to be able to run the `compiler` provided. - // - // Notably this munges the dynamic library lookup path to point to the - // right location to run `compiler`. - let mut lib_paths: Vec = vec![ - self.build.rustc_snapshot_libdir(), - self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), - ]; - - // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make - // mode) and that C compiler may need some extra PATH modification. Do - // so here. - if compiler.host.is_msvc() { - let curpaths = env::var_os("PATH").unwrap_or_default(); - let curpaths = env::split_paths(&curpaths).collect::>(); - for (k, v) in self.cc.borrow()[&compiler.host].env() { - if k != "PATH" { - continue; - } - for path in env::split_paths(v) { - if !curpaths.contains(&path) { - lib_paths.push(path); - } - } - } - } - - add_dylib_path(lib_paths, &mut cmd); - - // Provide a RUSTC for this command to use. - cmd.env("RUSTC", &self.initial_rustc); - - cmd - } -} diff --git a/standalonex/src/src/core/build_steps/toolstate.rs b/standalonex/src/src/core/build_steps/toolstate.rs deleted file mode 100644 index 8ac311b2..00000000 --- a/standalonex/src/src/core/build_steps/toolstate.rs +++ /dev/null @@ -1,459 +0,0 @@ -//! [Toolstate] checks to keep tools building -//! -//! Reachable via `./x.py test` but mostly relevant for CI, since it isn't run locally by default. -//! -//! [Toolstate]: https://forge.rust-lang.org/infra/toolstate.html - -use std::collections::HashMap; -use std::io::{Seek, SeekFrom}; -use std::path::{Path, PathBuf}; -use std::{env, fmt, fs, time}; - -use serde_derive::{Deserialize, Serialize}; - -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::utils::helpers::{self, t}; - -// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. -const BETA_WEEK_START: u64 = 35; - -#[cfg(target_os = "linux")] -const OS: Option<&str> = Some("linux"); - -#[cfg(windows)] -const OS: Option<&str> = Some("windows"); - -#[cfg(all(not(target_os = "linux"), not(windows)))] -const OS: Option<&str> = None; - -type ToolstateData = HashMap, ToolState>; - -#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(rename_all = "kebab-case")] -/// Whether a tool can be compiled, tested or neither -pub enum ToolState { - /// The tool compiles successfully, but the test suite fails - TestFail = 1, - /// The tool compiles successfully and its test suite passes - TestPass = 2, - /// The tool can't even be compiled - BuildFail = 0, -} - -impl fmt::Display for ToolState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", match self { - ToolState::TestFail => "test-fail", - ToolState::TestPass => "test-pass", - ToolState::BuildFail => "build-fail", - }) - } -} - -/// Number of days after the last promotion of beta. -/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. -/// The Wednesday after this has value 0. -/// We track this value to prevent regressing tools in the last week of the 6-week cycle. -fn days_since_beta_promotion() -> u64 { - let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); - (since_epoch.as_secs() / 86400 - 20) % 42 -} - -// These tools must test-pass on the beta/stable channels. -// -// On the nightly channel, their build step must be attempted, but they may not -// be able to build successfully. -static STABLE_TOOLS: &[(&str, &str)] = &[ - ("book", "src/doc/book"), - ("nomicon", "src/doc/nomicon"), - ("reference", "src/doc/reference"), - ("rust-by-example", "src/doc/rust-by-example"), - ("edition-guide", "src/doc/edition-guide"), -]; - -// These tools are permitted to not build on the beta/stable channels. -// -// We do require that we checked whether they build or not on the tools builder, -// though, as otherwise we will be unable to file an issue if they start -// failing. -static NIGHTLY_TOOLS: &[(&str, &str)] = &[ - ("embedded-book", "src/doc/embedded-book"), - // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), -]; - -fn print_error(tool: &str, submodule: &str) { - eprintln!(); - eprintln!("We detected that this PR updated '{tool}', but its tests failed."); - eprintln!(); - eprintln!("If you do intend to update '{tool}', please check the error messages above and"); - eprintln!("commit another update."); - eprintln!(); - eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); - eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); - eprintln!("proper steps."); - crate::exit!(3); -} - -fn check_changed_files(builder: &Builder<'_>, toolstates: &HashMap, ToolState>) { - // Changed files - let output = helpers::git(None) - .arg("diff") - .arg("--name-status") - .arg("HEAD") - .arg("HEAD^") - .run_capture(builder) - .stdout(); - - for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); - eprintln!("Verifying status of {tool}..."); - if !changed { - continue; - } - - eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); - if toolstates[*tool] != ToolState::TestPass { - print_error(tool, submodule); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ToolStateCheck; - -impl Step for ToolStateCheck { - type Output = (); - - /// Checks tool state status. - /// - /// This is intended to be used in the `checktools.sh` script. To use - /// this, set `save-toolstates` in `config.toml` so that tool status will - /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all - /// of the tools to populate the JSON file. After that is done, this - /// command can be run to check for any status failures, and exits with an - /// error if there are any. - /// - /// This also handles publishing the results to the `history` directory of - /// the toolstate repo - /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a - /// *separate* step of updating the `latest.json` file and creating GitHub - /// issues and comments in `src/ci/publish_toolstate.sh`, which is only - /// performed on master. (The shell/python code is intended to be migrated - /// here eventually.) - /// - /// The rules for failure are: - /// * If the PR modifies a tool, the status must be test-pass. - /// NOTE: There is intent to change this, see - /// . - /// * All "stable" tools must be test-pass on the stable or beta branches. - /// * During beta promotion week, a PR is not allowed to "regress" a - /// stable tool. That is, the status is not allowed to get worse - /// (test-pass to test-fail or build-fail). - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - - let days_since_beta_promotion = days_since_beta_promotion(); - let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; - let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); - let toolstates = builder.toolstates(); - - let mut did_error = false; - - for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - if !toolstates.contains_key(*tool) { - did_error = true; - eprintln!("ERROR: Tool `{tool}` was not recorded in tool state."); - } - } - - if did_error { - crate::exit!(1); - } - - check_changed_files(builder, &toolstates); - checkout_toolstate_repo(builder); - let old_toolstate = read_old_toolstate(); - - for (tool, _) in STABLE_TOOLS.iter() { - let state = toolstates[*tool]; - - if state != ToolState::TestPass { - if !is_nightly { - did_error = true; - eprintln!("ERROR: Tool `{tool}` should be test-pass but is {state}"); - } else if in_beta_week { - let old_state = old_toolstate - .iter() - .find(|ts| ts.tool == *tool) - .expect("latest.json missing tool") - .state(); - if state < old_state { - did_error = true; - eprintln!( - "ERROR: Tool `{tool}` has regressed from {old_state} to {state} during beta week." - ); - } else { - // This warning only appears in the logs, which most - // people won't read. It's mostly here for testing and - // debugging. - eprintln!( - "WARNING: Tool `{tool}` is not test-pass (is `{state}`), \ - this should be fixed before beta is branched." - ); - } - } - // `publish_toolstate.py` is responsible for updating - // `latest.json` and creating comments/issues warning people - // if there is a regression. That all happens in a separate CI - // job on the master branch once the PR has passed all tests - // on the `auto` branch. - } - } - - if did_error { - crate::exit!(1); - } - - if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { - commit_toolstate_change(builder, &toolstates); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("check-tools") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ToolStateCheck); - } -} - -impl Builder<'_> { - fn toolstates(&self) -> HashMap, ToolState> { - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = t!(fs::OpenOptions::new() - .create(true) - .truncate(false) - .write(true) - .read(true) - .open(path)); - - serde_json::from_reader(&mut file).unwrap_or_default() - } else { - Default::default() - } - } - - /// Updates the actual toolstate of a tool. - /// - /// The toolstates are saved to the file specified by the key - /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be - /// done. The file is updated immediately after this function completes. - pub fn save_toolstate(&self, tool: &str, state: ToolState) { - use std::io::Write; - - // If we're in a dry run setting we don't want to save toolstates as - // that means if we e.g. panic down the line it'll look like we tested - // everything (but we actually haven't). - if self.config.dry_run() { - return; - } - // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking - // in all the places we could save toolstate and just do so here. - if tool == "clippy-driver" || tool == "rustfmt" { - return; - } - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = t!(fs::OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(path)); - - let mut current_toolstates: HashMap, ToolState> = - serde_json::from_reader(&mut file).unwrap_or_default(); - current_toolstates.insert(tool.into(), state); - t!(file.seek(SeekFrom::Start(0))); - t!(file.set_len(0)); - t!(serde_json::to_writer(&file, ¤t_toolstates)); - t!(writeln!(file)); // make sure this ends in a newline - } - } -} - -fn toolstate_repo() -> String { - env::var("TOOLSTATE_REPO") - .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) -} - -/// Directory where the toolstate repo is checked out. -const TOOLSTATE_DIR: &str = "rust-toolstate"; - -/// Checks out the toolstate repo into `TOOLSTATE_DIR`. -fn checkout_toolstate_repo(builder: &Builder<'_>) { - if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { - prepare_toolstate_config(builder, &token); - } - if Path::new(TOOLSTATE_DIR).exists() { - eprintln!("Cleaning old toolstate directory..."); - t!(fs::remove_dir_all(TOOLSTATE_DIR)); - } - - helpers::git(None) - .arg("clone") - .arg("--depth=1") - .arg(toolstate_repo()) - .arg(TOOLSTATE_DIR) - .run(builder); -} - -/// Sets up config and authentication for modifying the toolstate repo. -fn prepare_toolstate_config(builder: &Builder<'_>, token: &str) { - fn git_config(builder: &Builder<'_>, key: &str, value: &str) { - helpers::git(None).arg("config").arg("--global").arg(key).arg(value).run(builder); - } - - // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date - // as well. - git_config(builder, "user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); - git_config(builder, "user.name", "Rust Toolstate Update"); - git_config(builder, "credential.helper", "store"); - - let credential = format!("https://{token}:x-oauth-basic@github.com\n",); - let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); - t!(fs::write(git_credential_path, credential)); -} - -/// Reads the latest toolstate from the toolstate repo. -fn read_old_toolstate() -> Vec { - let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); - let old_toolstate = t!(fs::read(latest_path)); - t!(serde_json::from_slice(&old_toolstate)) -} - -/// This function `commit_toolstate_change` provides functionality for pushing a change -/// to the `rust-toolstate` repository. -/// -/// The function relies on a GitHub bot user, which should have a Personal access -/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for -/// some reason you need to change the token, please update the Azure Pipelines -/// variable group. -/// -/// 1. Generate a new Personal access token: -/// -/// * Login to the bot account, and go to Settings -> Developer settings -> -/// Personal access tokens -/// * Click "Generate new token" -/// * Enable the "public_repo" permission, then click "Generate token" -/// * Copy the generated token (should be a 40-digit hexadecimal number). -/// Save it somewhere secure, as the token would be gone once you leave -/// the page. -/// -/// 2. Update the variable group in Azure Pipelines -/// -/// * Ping a member of the infrastructure team to do this. -/// -/// 4. Replace the email address below if the bot account identity is changed -/// -/// * See -/// if a private email by GitHub is wanted. -fn commit_toolstate_change(builder: &Builder<'_>, current_toolstate: &ToolstateData) { - let message = format!("({} CI update)", OS.expect("linux/windows only")); - let mut success = false; - for _ in 1..=5 { - // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. - // This does *not* change the "current toolstate"; that only happens post-landing - // via `src/ci/docker/publish_toolstate.sh`. - publish_test_results(builder, current_toolstate); - - // `git commit` failing means nothing to commit. - let status = helpers::git(Some(Path::new(TOOLSTATE_DIR))) - .allow_failure() - .arg("commit") - .arg("-a") - .arg("-m") - .arg(&message) - .run(builder); - if !status { - success = true; - break; - } - - let status = helpers::git(Some(Path::new(TOOLSTATE_DIR))) - .allow_failure() - .arg("push") - .arg("origin") - .arg("master") - .run(builder); - // If we successfully push, exit. - if status { - success = true; - break; - } - eprintln!("Sleeping for 3 seconds before retrying push"); - std::thread::sleep(std::time::Duration::from_secs(3)); - helpers::git(Some(Path::new(TOOLSTATE_DIR))) - .arg("fetch") - .arg("origin") - .arg("master") - .run(builder); - helpers::git(Some(Path::new(TOOLSTATE_DIR))) - .arg("reset") - .arg("--hard") - .arg("origin/master") - .run(builder); - } - - if !success { - panic!("Failed to update toolstate repository with new data"); - } -} - -/// Updates the "history" files with the latest results. -/// -/// These results will later be promoted to `latest.json` by the -/// `publish_toolstate.py` script if the PR passes all tests and is merged to -/// master. -fn publish_test_results(builder: &Builder<'_>, current_toolstate: &ToolstateData) { - let commit = helpers::git(None).arg("rev-parse").arg("HEAD").run_capture(builder).stdout(); - - let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); - - let history_path = Path::new(TOOLSTATE_DIR) - .join("history") - .join(format!("{}.tsv", OS.expect("linux/windows only"))); - let mut file = t!(fs::read_to_string(&history_path)); - let end_of_first_line = file.find('\n').unwrap(); - file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); - t!(fs::write(&history_path, file)); -} - -#[derive(Debug, Deserialize)] -struct RepoState { - tool: String, - windows: ToolState, - linux: ToolState, -} - -impl RepoState { - fn state(&self) -> ToolState { - if cfg!(target_os = "linux") { - self.linux - } else if cfg!(windows) { - self.windows - } else { - unimplemented!() - } - } -} diff --git a/standalonex/src/src/core/build_steps/vendor.rs b/standalonex/src/src/core/build_steps/vendor.rs deleted file mode 100644 index 82a6b4d4..00000000 --- a/standalonex/src/src/core/build_steps/vendor.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::path::PathBuf; - -use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK; -use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::utils::exec::command; - -/// List of default paths used for vendoring for `x vendor` and dist tarballs. -pub fn default_paths_to_vendor(builder: &Builder<'_>) -> Vec { - let mut paths = vec![]; - for p in [ - "src/tools/cargo/Cargo.toml", - "src/tools/rust-analyzer/Cargo.toml", - "compiler/rustc_codegen_cranelift/Cargo.toml", - "compiler/rustc_codegen_gcc/Cargo.toml", - "library/Cargo.toml", - "src/bootstrap/Cargo.toml", - "src/tools/rustbook/Cargo.toml", - "src/tools/rustc-perf/Cargo.toml", - "src/tools/opt-dist/Cargo.toml", - ] { - paths.push(builder.src.join(p)); - } - - paths -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub(crate) struct Vendor { - sync_args: Vec, - versioned_dirs: bool, - root_dir: PathBuf, -} - -impl Step for Vendor { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("placeholder").default_condition(true) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Vendor { - sync_args: run.builder.config.cmd.vendor_sync_args(), - versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(), - root_dir: run.builder.src.clone(), - }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = command(&builder.initial_cargo); - cmd.arg("vendor"); - - if self.versioned_dirs { - cmd.arg("--versioned-dirs"); - } - - // These submodules must be present for `x vendor` to work. - for submodule in SUBMODULES_FOR_RUSTBOOK.iter().chain(["src/tools/cargo"].iter()) { - builder.build.require_submodule(submodule, None); - } - - // Sync these paths by default. - for p in default_paths_to_vendor(builder) { - cmd.arg("--sync").arg(p); - } - - // Also sync explicitly requested paths. - for sync_arg in self.sync_args { - cmd.arg("--sync").arg(sync_arg); - } - - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - cmd.env("RUSTC_BOOTSTRAP", "1"); - - cmd.current_dir(self.root_dir); - - cmd.run(builder); - } -} diff --git a/standalonex/src/src/core/builder/cargo.rs b/standalonex/src/src/core/builder/cargo.rs deleted file mode 100644 index 0688a1d6..00000000 --- a/standalonex/src/src/core/builder/cargo.rs +++ /dev/null @@ -1,1211 +0,0 @@ -use std::env; -use std::ffi::{OsStr, OsString}; -use std::path::{Path, PathBuf}; - -use super::{Builder, Kind}; -use crate::core::build_steps::tool::SourceType; -use crate::core::build_steps::{compile, test}; -use crate::core::config::SplitDebuginfo; -use crate::core::config::flags::Color; -use crate::utils::helpers::{ - self, LldThreads, add_link_lib_path, check_cfg_arg, linker_args, linker_flags, -}; -use crate::{ - BootstrapCommand, CLang, Compiler, DocTests, DryRun, EXTRA_CHECK_CFGS, GitRepo, Mode, - TargetSelection, command, prepare_behaviour_dump_dir, t, -}; - -/// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler -/// later. -/// -/// `-Z crate-attr` flags will be applied recursively on the target code using the -/// `rustc_parse::parser::Parser`. See `rustc_builtin_macros::cmdline_attrs::inject` for more -/// information. -#[derive(Debug, Clone)] -struct Rustflags(String, TargetSelection); - -impl Rustflags { - fn new(target: TargetSelection) -> Rustflags { - let mut ret = Rustflags(String::new(), target); - ret.propagate_cargo_env("RUSTFLAGS"); - ret - } - - /// By default, cargo will pick up on various variables in the environment. However, bootstrap - /// reuses those variables to pass additional flags to rustdoc, so by default they get - /// overridden. Explicitly add back any previous value in the environment. - /// - /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. - fn propagate_cargo_env(&mut self, prefix: &str) { - // Inherit `RUSTFLAGS` by default ... - self.env(prefix); - - // ... and also handle target-specific env RUSTFLAGS if they're configured. - let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); - self.env(&target_specific); - } - - fn env(&mut self, env: &str) { - if let Ok(s) = env::var(env) { - for part in s.split(' ') { - self.arg(part); - } - } - } - - fn arg(&mut self, arg: &str) -> &mut Self { - assert_eq!(arg.split(' ').count(), 1); - if !self.0.is_empty() { - self.0.push(' '); - } - self.0.push_str(arg); - self - } -} - -/// Flags that are passed to the `rustc` shim binary. These flags will only be applied when -/// compiling host code, i.e. when `--target` is unset. -#[derive(Debug, Default)] -struct HostFlags { - rustc: Vec, -} - -impl HostFlags { - const SEPARATOR: &'static str = " "; - - /// Adds a host rustc flag. - fn arg>(&mut self, flag: S) { - let value = flag.into().trim().to_string(); - assert!(!value.contains(Self::SEPARATOR)); - self.rustc.push(value); - } - - /// Encodes all the flags into a single string. - fn encode(self) -> String { - self.rustc.join(Self::SEPARATOR) - } -} - -#[derive(Debug)] -pub struct Cargo { - command: BootstrapCommand, - compiler: Compiler, - target: TargetSelection, - rustflags: Rustflags, - rustdocflags: Rustflags, - hostflags: HostFlags, - allow_features: String, -} - -impl Cargo { - /// Calls [`Builder::cargo`] and [`Cargo::configure_linker`] to prepare an invocation of `cargo` - /// to be run. - pub fn new( - builder: &Builder<'_>, - compiler: Compiler, - mode: Mode, - source_type: SourceType, - target: TargetSelection, - cmd_kind: Kind, - ) -> Cargo { - let mut cargo = builder.cargo(compiler, mode, source_type, target, cmd_kind); - - match cmd_kind { - // No need to configure the target linker for these command types, - // as they don't invoke rustc at all. - Kind::Clean | Kind::Suggest | Kind::Format | Kind::Setup => {} - _ => { - cargo.configure_linker(builder); - } - } - - cargo - } - - pub fn into_cmd(self) -> BootstrapCommand { - self.into() - } - - /// Same as [`Cargo::new`] except this one doesn't configure the linker with - /// [`Cargo::configure_linker`]. - pub fn new_for_mir_opt_tests( - builder: &Builder<'_>, - compiler: Compiler, - mode: Mode, - source_type: SourceType, - target: TargetSelection, - cmd_kind: Kind, - ) -> Cargo { - builder.cargo(compiler, mode, source_type, target, cmd_kind) - } - - pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { - self.rustdocflags.arg(arg); - self - } - - pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { - self.rustflags.arg(arg); - self - } - - pub fn arg(&mut self, arg: impl AsRef) -> &mut Cargo { - self.command.arg(arg.as_ref()); - self - } - - pub fn args(&mut self, args: I) -> &mut Cargo - where - I: IntoIterator, - S: AsRef, - { - for arg in args { - self.arg(arg.as_ref()); - } - self - } - - /// Add an env var to the cargo command instance. Note that `RUSTFLAGS`/`RUSTDOCFLAGS` must go - /// through [`Cargo::rustdocflags`] and [`Cargo::rustflags`] because inconsistent `RUSTFLAGS` - /// and `RUSTDOCFLAGS` usages will trigger spurious rebuilds. - pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Cargo { - assert_ne!(key.as_ref(), "RUSTFLAGS"); - assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); - self.command.env(key.as_ref(), value.as_ref()); - self - } - - pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>) { - builder.add_rustc_lib_path(self.compiler, &mut self.command); - } - - pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { - self.command.current_dir(dir); - self - } - - /// Adds nightly-only features that this invocation is allowed to use. - /// - /// By default, all nightly features are allowed. Once this is called, it will be restricted to - /// the given set. - pub fn allow_features(&mut self, features: &str) -> &mut Cargo { - if !self.allow_features.is_empty() { - self.allow_features.push(','); - } - self.allow_features.push_str(features); - self - } - - fn configure_linker(&mut self, builder: &Builder<'_>) -> &mut Cargo { - let target = self.target; - let compiler = self.compiler; - - // Dealing with rpath here is a little special, so let's go into some - // detail. First off, `-rpath` is a linker option on Unix platforms - // which adds to the runtime dynamic loader path when looking for - // dynamic libraries. We use this by default on Unix platforms to ensure - // that our nightlies behave the same on Windows, that is they work out - // of the box. This can be disabled by setting `rpath = false` in `[rust]` - // table of `config.toml` - // - // Ok, so the astute might be wondering "why isn't `-C rpath` used - // here?" and that is indeed a good question to ask. This codegen - // option is the compiler's current interface to generating an rpath. - // Unfortunately it doesn't quite suffice for us. The flag currently - // takes no value as an argument, so the compiler calculates what it - // should pass to the linker as `-rpath`. This unfortunately is based on - // the **compile time** directory structure which when building with - // Cargo will be very different than the runtime directory structure. - // - // All that's a really long winded way of saying that if we use - // `-Crpath` then the executables generated have the wrong rpath of - // something like `$ORIGIN/deps` when in fact the way we distribute - // rustc requires the rpath to be `$ORIGIN/../lib`. - // - // So, all in all, to set up the correct rpath we pass the linker - // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it - // fun to pass a flag to a tool to pass a flag to pass a flag to a tool - // to change a flag in a binary? - if builder.config.rpath_enabled(target) && helpers::use_host_linker(target) { - let libdir = builder.sysroot_libdir_relative(compiler).to_str().unwrap(); - let rpath = if target.contains("apple") { - // Note that we need to take one extra step on macOS to also pass - // `-Wl,-instal_name,@rpath/...` to get things to work right. To - // do that we pass a weird flag to the compiler to get it to do - // so. Note that this is definitely a hack, and we should likely - // flesh out rpath support more fully in the future. - self.rustflags.arg("-Zosx-rpath-install-name"); - Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) - } else if !target.is_windows() && !target.contains("aix") && !target.contains("xous") { - self.rustflags.arg("-Clink-args=-Wl,-z,origin"); - Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) - } else { - None - }; - if let Some(rpath) = rpath { - self.rustflags.arg(&format!("-Clink-args={rpath}")); - } - } - - for arg in linker_args(builder, compiler.host, LldThreads::Yes) { - self.hostflags.arg(&arg); - } - - if let Some(target_linker) = builder.linker(target) { - let target = crate::envify(&target.triple); - self.command.env(format!("CARGO_TARGET_{target}_LINKER"), target_linker); - } - // We want to set -Clinker using Cargo, therefore we only call `linker_flags` and not - // `linker_args` here. - for flag in linker_flags(builder, target, LldThreads::Yes) { - self.rustflags.arg(&flag); - } - for arg in linker_args(builder, target, LldThreads::Yes) { - self.rustdocflags.arg(&arg); - } - - if !builder.config.dry_run() - && builder.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") - { - self.rustflags.arg("-Clink-arg=-gz"); - } - - // Throughout the build Cargo can execute a number of build scripts - // compiling C/C++ code and we need to pass compilers, archivers, flags, etc - // obtained previously to those build scripts. - // Build scripts use either the `cc` crate or `configure/make` so we pass - // the options through environment variables that are fetched and understood by both. - // - // FIXME: the guard against msvc shouldn't need to be here - if target.is_msvc() { - if let Some(ref cl) = builder.config.llvm_clang_cl { - // FIXME: There is a bug in Clang 18 when building for ARM64: - // https://github.com/llvm/llvm-project/pull/81849. This is - // fixed in LLVM 19, but can't be backported. - if !target.starts_with("aarch64") && !target.starts_with("arm64ec") { - self.command.env("CC", cl).env("CXX", cl); - } - } - } else { - let ccache = builder.config.ccache.as_ref(); - let ccacheify = |s: &Path| { - let ccache = match ccache { - Some(ref s) => s, - None => return s.display().to_string(), - }; - // FIXME: the cc-rs crate only recognizes the literal strings - // `ccache` and `sccache` when doing caching compilations, so we - // mirror that here. It should probably be fixed upstream to - // accept a new env var or otherwise work with custom ccache - // vars. - match &ccache[..] { - "ccache" | "sccache" => format!("{} {}", ccache, s.display()), - _ => s.display().to_string(), - } - }; - let triple_underscored = target.triple.replace('-', "_"); - let cc = ccacheify(&builder.cc(target)); - self.command.env(format!("CC_{triple_underscored}"), &cc); - - let cflags = builder.cflags(target, GitRepo::Rustc, CLang::C).join(" "); - self.command.env(format!("CFLAGS_{triple_underscored}"), &cflags); - - if let Some(ar) = builder.ar(target) { - let ranlib = format!("{} s", ar.display()); - self.command - .env(format!("AR_{triple_underscored}"), ar) - .env(format!("RANLIB_{triple_underscored}"), ranlib); - } - - if let Ok(cxx) = builder.cxx(target) { - let cxx = ccacheify(&cxx); - let cxxflags = builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); - self.command - .env(format!("CXX_{triple_underscored}"), &cxx) - .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); - } - } - - self - } -} - -impl From for BootstrapCommand { - fn from(mut cargo: Cargo) -> BootstrapCommand { - let rustflags = &cargo.rustflags.0; - if !rustflags.is_empty() { - cargo.command.env("RUSTFLAGS", rustflags); - } - - let rustdocflags = &cargo.rustdocflags.0; - if !rustdocflags.is_empty() { - cargo.command.env("RUSTDOCFLAGS", rustdocflags); - } - - let encoded_hostflags = cargo.hostflags.encode(); - if !encoded_hostflags.is_empty() { - cargo.command.env("RUSTC_HOST_FLAGS", encoded_hostflags); - } - - if !cargo.allow_features.is_empty() { - cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); - } - cargo.command - } -} - -impl Builder<'_> { - /// Like [`Builder::cargo`], but only passes flags that are valid for all commands. - pub fn bare_cargo( - &self, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - cmd_kind: Kind, - ) -> BootstrapCommand { - let mut cargo = match cmd_kind { - Kind::Clippy => { - let mut cargo = self.cargo_clippy_cmd(compiler); - cargo.arg(cmd_kind.as_str()); - cargo - } - Kind::MiriSetup => { - let mut cargo = self.cargo_miri_cmd(compiler); - cargo.arg("miri").arg("setup"); - cargo - } - Kind::MiriTest => { - let mut cargo = self.cargo_miri_cmd(compiler); - cargo.arg("miri").arg("test"); - cargo - } - _ => { - let mut cargo = command(&self.initial_cargo); - cargo.arg(cmd_kind.as_str()); - cargo - } - }; - - // Run cargo from the source root so it can find .cargo/config. - // This matters when using vendoring and the working directory is outside the repository. - cargo.current_dir(&self.src); - - let out_dir = self.stage_out(compiler, mode); - cargo.env("CARGO_TARGET_DIR", &out_dir); - - // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` - // from out of tree it shouldn't matter, since x.py is only used for - // building in-tree. - let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; - match self.build.config.color { - Color::Always => { - cargo.arg("--color=always"); - for log in &color_logs { - cargo.env(log, "always"); - } - } - Color::Never => { - cargo.arg("--color=never"); - for log in &color_logs { - cargo.env(log, "never"); - } - } - Color::Auto => {} // nothing to do - } - - if cmd_kind != Kind::Install { - cargo.arg("--target").arg(target.rustc_target_arg()); - } else { - assert_eq!(target, compiler.host); - } - - if self.config.rust_optimize.is_release() && - // cargo bench/install do not accept `--release` and miri doesn't want it - !matches!(cmd_kind, Kind::Bench | Kind::Install | Kind::Miri | Kind::MiriSetup | Kind::MiriTest) - { - cargo.arg("--release"); - } - - // Remove make-related flags to ensure Cargo can correctly set things up - cargo.env_remove("MAKEFLAGS"); - cargo.env_remove("MFLAGS"); - - cargo - } - - /// This will create a [`BootstrapCommand`] that represents a pending execution of cargo. This - /// cargo will be configured to use `compiler` as the actual rustc compiler, its output will be - /// scoped by `mode`'s output directory, it will pass the `--target` flag for the specified - /// `target`, and will be executing the Cargo command `cmd`. `cmd` can be `miri-cmd` for - /// commands to be run with Miri. - fn cargo( - &self, - compiler: Compiler, - mode: Mode, - source_type: SourceType, - target: TargetSelection, - cmd_kind: Kind, - ) -> Cargo { - let mut cargo = self.bare_cargo(compiler, mode, target, cmd_kind); - let out_dir = self.stage_out(compiler, mode); - - let mut hostflags = HostFlags::default(); - - // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, - // so we need to explicitly clear out if they've been updated. - for backend in self.codegen_backends(compiler) { - self.clear_if_dirty(&out_dir, &backend); - } - - if cmd_kind == Kind::Doc { - let my_out = match mode { - // This is the intended out directory for compiler documentation. - Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), - Mode::Std => { - if self.config.cmd.json() { - out_dir.join(target).join("json-doc") - } else { - out_dir.join(target).join("doc") - } - } - _ => panic!("doc mode {mode:?} not expected"), - }; - let rustdoc = self.rustdoc(compiler); - self.clear_if_dirty(&my_out, &rustdoc); - } - - let profile_var = |name: &str| { - let profile = if self.config.rust_optimize.is_release() { "RELEASE" } else { "DEV" }; - format!("CARGO_PROFILE_{}_{}", profile, name) - }; - - // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config - // needs to not accidentally link to libLLVM in stage0/lib. - cargo.env("REAL_LIBRARY_PATH_VAR", helpers::dylib_path_var()); - if let Some(e) = env::var_os(helpers::dylib_path_var()) { - cargo.env("REAL_LIBRARY_PATH", e); - } - - // Set a flag for `check`/`clippy`/`fix`, so that certain build - // scripts can do less work (i.e. not building/requiring LLVM). - if matches!(cmd_kind, Kind::Check | Kind::Clippy | Kind::Fix) { - // If we've not yet built LLVM, or it's stale, then bust - // the rustc_llvm cache. That will always work, even though it - // may mean that on the next non-check build we'll need to rebuild - // rustc_llvm. But if LLVM is stale, that'll be a tiny amount - // of work comparatively, and we'd likely need to rebuild it anyway, - // so that's okay. - if crate::core::build_steps::llvm::prebuilt_llvm_config(self, target, false) - .should_build() - { - cargo.env("RUST_CHECK", "1"); - } - } - - let stage = if compiler.stage == 0 && self.local_rebuild { - // Assume the local-rebuild rustc already has stage1 features. - 1 - } else { - compiler.stage - }; - - // We synthetically interpret a stage0 compiler used to build tools as a - // "raw" compiler in that it's the exact snapshot we download. Normally - // the stage0 build means it uses libraries build by the stage0 - // compiler, but for tools we just use the precompiled libraries that - // we've downloaded - let use_snapshot = mode == Mode::ToolBootstrap; - assert!(!use_snapshot || stage == 0 || self.local_rebuild); - - let maybe_sysroot = self.sysroot(compiler); - let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; - let libdir = self.rustc_libdir(compiler); - - let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8"); - if self.is_verbose() && !matches!(self.config.dry_run, DryRun::SelfCheck) { - println!("using sysroot {sysroot_str}"); - } - - let mut rustflags = Rustflags::new(target); - if stage != 0 { - if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); - } else { - if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_BOOTSTRAP"); - rustflags.arg("--cfg=bootstrap"); - } - - if cmd_kind == Kind::Clippy { - // clippy overwrites sysroot if we pass it to cargo. - // Pass it directly to clippy instead. - // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, - // so it has no way of knowing the sysroot. - rustflags.arg("--sysroot"); - rustflags.arg(sysroot_str); - } - - let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { - Some(setting) => { - // If an explicit setting is given, use that - setting - } - None => { - if mode == Mode::Std { - // The standard library defaults to the legacy scheme - false - } else { - // The compiler and tools default to the new scheme - true - } - } - }; - - // By default, windows-rs depends on a native library that doesn't get copied into the - // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native - // library unnecessary. This can be removed when windows-rs enables raw-dylib - // unconditionally. - if let Mode::Rustc | Mode::ToolRustc = mode { - rustflags.arg("--cfg=windows_raw_dylib"); - } - - if use_new_symbol_mangling { - rustflags.arg("-Csymbol-mangling-version=v0"); - } else { - rustflags.arg("-Csymbol-mangling-version=legacy"); - } - - // FIXME: the following components don't build with `-Zrandomize-layout` yet: - // - wasm-component-ld, due to the `wast`crate - // - rust-analyzer, due to the rowan crate - // so we exclude entire categories of steps here due to lack of fine-grained control over - // rustflags. - if self.config.rust_randomize_layout && mode != Mode::ToolStd && mode != Mode::ToolRustc { - rustflags.arg("-Zrandomize-layout"); - } - - // Enable compile-time checking of `cfg` names, values and Cargo `features`. - // - // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like - // backtrace, core_simd, std_float, ...), those dependencies have their own - // features but cargo isn't involved in the #[path] process and so cannot pass the - // complete list of features, so for that reason we don't enable checking of - // features for std crates. - if mode == Mode::Std { - rustflags.arg("--check-cfg=cfg(feature,values(any()))"); - } - - // Add extra cfg not defined in/by rustc - // - // Note: Although it would seems that "-Zunstable-options" to `rustflags` is useless as - // cargo would implicitly add it, it was discover that sometimes bootstrap only use - // `rustflags` without `cargo` making it required. - rustflags.arg("-Zunstable-options"); - for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { - if restricted_mode.is_none() || *restricted_mode == Some(mode) { - rustflags.arg(&check_cfg_arg(name, *values)); - } - } - - // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments - // to the host invocation here, but rather Cargo should know what flags to pass rustc - // itself. - if stage == 0 { - hostflags.arg("--cfg=bootstrap"); - } - // Cargo doesn't pass RUSTFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - hostflags.arg("-Zunstable-options"); - hostflags.arg("--check-cfg=cfg(bootstrap)"); - - // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, - // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See - // #71458. - let mut rustdocflags = rustflags.clone(); - rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); - if stage == 0 { - rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); - } else { - rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); - } - - if let Ok(s) = env::var("CARGOFLAGS") { - cargo.args(s.split_whitespace()); - } - - match mode { - Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} - Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { - // Build proc macros both for the host and the target unless proc-macros are not - // supported by the target. - if target != compiler.host && cmd_kind != Kind::Check { - let error = command(self.rustc(compiler)) - .arg("--target") - .arg(target.rustc_target_arg()) - .arg("--print=file-names") - .arg("--crate-type=proc-macro") - .arg("-") - .run_capture(self) - .stderr(); - let not_supported = error - .lines() - .any(|line| line.contains("unsupported crate type `proc-macro`")); - if !not_supported { - cargo.arg("-Zdual-proc-macros"); - rustflags.arg("-Zdual-proc-macros"); - } - } - } - } - - // This tells Cargo (and in turn, rustc) to output more complete - // dependency information. Most importantly for bootstrap, this - // includes sysroot artifacts, like libstd, which means that we don't - // need to track those in bootstrap (an error prone process!). This - // feature is currently unstable as there may be some bugs and such, but - // it represents a big improvement in bootstrap's reliability on - // rebuilds, so we're using it here. - // - // For some additional context, see #63470 (the PR originally adding - // this), as well as #63012 which is the tracking issue for this - // feature on the rustc side. - cargo.arg("-Zbinary-dep-depinfo"); - let allow_features = match mode { - Mode::ToolBootstrap | Mode::ToolStd => { - // Restrict the allowed features so we don't depend on nightly - // accidentally. - // - // binary-dep-depinfo is used by bootstrap itself for all - // compilations. - // - // Lots of tools depend on proc_macro2 and proc-macro-error. - // Those have build scripts which assume nightly features are - // available if the `rustc` version is "nighty" or "dev". See - // bin/rustc.rs for why that is a problem. Instead of labeling - // those features for each individual tool that needs them, - // just blanket allow them here. - // - // If this is ever removed, be sure to add something else in - // its place to keep the restrictions in place (or make a way - // to unset RUSTC_BOOTSTRAP). - "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" - .to_string() - } - Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), - }; - - cargo.arg("-j").arg(self.jobs().to_string()); - - // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 - // Force cargo to output binaries with disambiguating hashes in the name - let mut metadata = if compiler.stage == 0 { - // Treat stage0 like a special channel, whether it's a normal prior- - // release rustc or a local rebuild with the same version, so we - // never mix these libraries by accident. - "bootstrap".to_string() - } else { - self.config.channel.to_string() - }; - // We want to make sure that none of the dependencies between - // std/test/rustc unify with one another. This is done for weird linkage - // reasons but the gist of the problem is that if librustc, libtest, and - // libstd all depend on libc from crates.io (which they actually do) we - // want to make sure they all get distinct versions. Things get really - // weird if we try to unify all these dependencies right now, namely - // around how many times the library is linked in dynamic libraries and - // such. If rustc were a static executable or if we didn't ship dylibs - // this wouldn't be a problem, but we do, so it is. This is in general - // just here to make sure things build right. If you can remove this and - // things still build right, please do! - match mode { - Mode::Std => metadata.push_str("std"), - // When we're building rustc tools, they're built with a search path - // that contains things built during the rustc build. For example, - // bitflags is built during the rustc build, and is a dependency of - // rustdoc as well. We're building rustdoc in a different target - // directory, though, which means that Cargo will rebuild the - // dependency. When we go on to build rustdoc, we'll look for - // bitflags, and find two different copies: one built during the - // rustc step and one that we just built. This isn't always a - // problem, somehow -- not really clear why -- but we know that this - // fixes things. - Mode::ToolRustc => metadata.push_str("tool-rustc"), - // Same for codegen backends. - Mode::Codegen => metadata.push_str("codegen"), - _ => {} - } - cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); - - if cmd_kind == Kind::Clippy { - rustflags.arg("-Zforce-unstable-if-unmarked"); - } - - rustflags.arg("-Zmacro-backtrace"); - - let want_rustdoc = self.doc_tests != DocTests::No; - - // Clear the output directory if the real rustc we're using has changed; - // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. - // - // Avoid doing this during dry run as that usually means the relevant - // compiler is not yet linked/copied properly. - // - // Only clear out the directory if we're compiling std; otherwise, we - // should let Cargo take care of things for us (via depdep info) - if !self.config.dry_run() && mode == Mode::Std && cmd_kind == Kind::Build { - self.clear_if_dirty(&out_dir, &self.rustc(compiler)); - } - - let rustdoc_path = match cmd_kind { - Kind::Doc | Kind::Test | Kind::MiriTest => self.rustdoc(compiler), - _ => PathBuf::from("/path/to/nowhere/rustdoc/not/required"), - }; - - // Customize the compiler we're running. Specify the compiler to cargo - // as our shim and then pass it some various options used to configure - // how the actual compiler itself is called. - // - // These variables are primarily all read by - // src/bootstrap/bin/{rustc.rs,rustdoc.rs} - cargo - .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) - .env("RUSTC_REAL", self.rustc(compiler)) - .env("RUSTC_STAGE", stage.to_string()) - .env("RUSTC_SYSROOT", sysroot) - .env("RUSTC_LIBDIR", libdir) - .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) - .env("RUSTDOC_REAL", rustdoc_path) - .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) - .env("RUSTC_BREAK_ON_ICE", "1"); - - // Set RUSTC_WRAPPER to the bootstrap shim, which switches between beta and in-tree - // sysroot depending on whether we're building build scripts. - // NOTE: we intentionally use RUSTC_WRAPPER so that we can support clippy - RUSTC is not - // respected by clippy-driver; RUSTC_WRAPPER happens earlier, before clippy runs. - cargo.env("RUSTC_WRAPPER", self.bootstrap_out.join("rustc")); - // NOTE: we also need to set RUSTC so cargo can run `rustc -vV`; apparently that ignores RUSTC_WRAPPER >:( - cargo.env("RUSTC", self.bootstrap_out.join("rustc")); - - // Someone might have set some previous rustc wrapper (e.g. - // sccache) before bootstrap overrode it. Respect that variable. - if let Some(existing_wrapper) = env::var_os("RUSTC_WRAPPER") { - cargo.env("RUSTC_WRAPPER_REAL", existing_wrapper); - } - - // If this is for `miri-test`, prepare the sysroots. - if cmd_kind == Kind::MiriTest { - self.ensure(compile::Std::new(compiler, compiler.host)); - let host_sysroot = self.sysroot(compiler); - let miri_sysroot = test::Miri::build_miri_sysroot(self, compiler, target); - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", &host_sysroot); - } - - cargo.env(profile_var("STRIP"), self.config.rust_strip.to_string()); - - if let Some(stack_protector) = &self.config.rust_stack_protector { - rustflags.arg(&format!("-Zstack-protector={stack_protector}")); - } - - if !matches!(cmd_kind, Kind::Build | Kind::Check | Kind::Clippy | Kind::Fix) && want_rustdoc - { - cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); - } - - let debuginfo_level = match mode { - Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, - Mode::Std => self.config.rust_debuginfo_level_std, - Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { - self.config.rust_debuginfo_level_tools - } - }; - cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); - if let Some(opt_level) = &self.config.rust_optimize.get_opt_level() { - cargo.env(profile_var("OPT_LEVEL"), opt_level); - } - cargo.env( - profile_var("DEBUG_ASSERTIONS"), - if mode == Mode::Std { - self.config.std_debug_assertions.to_string() - } else { - self.config.rustc_debug_assertions.to_string() - }, - ); - cargo.env( - profile_var("OVERFLOW_CHECKS"), - if mode == Mode::Std { - self.config.rust_overflow_checks_std.to_string() - } else { - self.config.rust_overflow_checks.to_string() - }, - ); - - match self.config.split_debuginfo(target) { - SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), - SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), - SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), - }; - - if self.config.cmd.bless() { - // Bless `expect!` tests. - cargo.env("UPDATE_EXPECT", "1"); - } - - if !mode.is_tool() { - cargo.env("RUSTC_FORCE_UNSTABLE", "1"); - } - - if let Some(x) = self.crt_static(target) { - if x { - rustflags.arg("-Ctarget-feature=+crt-static"); - } else { - rustflags.arg("-Ctarget-feature=-crt-static"); - } - } - - if let Some(x) = self.crt_static(compiler.host) { - let sign = if x { "+" } else { "-" }; - hostflags.arg(format!("-Ctarget-feature={sign}crt-static")); - } - - if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { - let map = format!("{}={}", self.build.src.display(), map_to); - cargo.env("RUSTC_DEBUGINFO_MAP", map); - - // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, - // in order to opportunistically reverse it later. - cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); - } - - if self.config.rust_remap_debuginfo { - let mut env_var = OsString::new(); - if self.config.vendor { - let vendor = self.build.src.join("vendor"); - env_var.push(vendor); - env_var.push("=/rust/deps"); - } else { - let registry_src = t!(home::cargo_home()).join("registry").join("src"); - for entry in t!(std::fs::read_dir(registry_src)) { - if !env_var.is_empty() { - env_var.push("\t"); - } - env_var.push(t!(entry).path()); - env_var.push("=/rust/deps"); - } - } - cargo.env("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP", env_var); - } - - // Enable usage of unstable features - cargo.env("RUSTC_BOOTSTRAP", "1"); - - if self.config.dump_bootstrap_shims { - prepare_behaviour_dump_dir(self.build); - - cargo - .env("DUMP_BOOTSTRAP_SHIMS", self.build.out.join("bootstrap-shims-dump")) - .env("BUILD_OUT", &self.build.out) - .env("CARGO_HOME", t!(home::cargo_home())); - }; - - self.add_rust_test_threads(&mut cargo); - - // Almost all of the crates that we compile as part of the bootstrap may - // have a build script, including the standard library. To compile a - // build script, however, it itself needs a standard library! This - // introduces a bit of a pickle when we're compiling the standard - // library itself. - // - // To work around this we actually end up using the snapshot compiler - // (stage0) for compiling build scripts of the standard library itself. - // The stage0 compiler is guaranteed to have a libstd available for use. - // - // For other crates, however, we know that we've already got a standard - // library up and running, so we can use the normal compiler to compile - // build scripts in that situation. - if mode == Mode::Std { - cargo - .env("RUSTC_SNAPSHOT", &self.initial_rustc) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); - } else { - cargo - .env("RUSTC_SNAPSHOT", self.rustc(compiler)) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); - } - - // Tools that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate Cargo projects. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if mode == Mode::ToolRustc || mode == Mode::Codegen { - if let Some(llvm_config) = self.llvm_config(target) { - let llvm_libdir = - command(llvm_config).arg("--libdir").run_capture_stdout(self).stdout(); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); - } - } - - // Compile everything except libraries and proc macros with the more - // efficient initial-exec TLS model. This doesn't work with `dlopen`, - // so we can't use it by default in general, but we can use it for tools - // and our own internal libraries. - if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { - cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1"); - } - - // Ignore incremental modes except for stage0, since we're - // not guaranteeing correctness across builds if the compiler - // is changing under your feet. - if self.config.incremental && compiler.stage == 0 { - cargo.env("CARGO_INCREMENTAL", "1"); - } else { - // Don't rely on any default setting for incr. comp. in Cargo - cargo.env("CARGO_INCREMENTAL", "0"); - } - - if let Some(ref on_fail) = self.config.on_fail { - cargo.env("RUSTC_ON_FAIL", on_fail); - } - - if self.config.print_step_timings { - cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); - } - - if self.config.print_step_rusage { - cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); - } - - if self.config.backtrace_on_ice { - cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); - } - - if self.is_verbose() { - // This provides very useful logs especially when debugging build cache-related stuff. - cargo.env("CARGO_LOG", "cargo::core::compiler::fingerprint=info"); - } - - cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); - - // Downstream forks of the Rust compiler might want to use a custom libc to add support for - // targets that are not yet available upstream. Adding a patch to replace libc with a - // custom one would cause compilation errors though, because Cargo would interpret the - // custom libc as part of the workspace, and apply the check-cfg lints on it. - // - // The libc build script emits check-cfg flags only when this environment variable is set, - // so this line allows the use of custom libcs. - cargo.env("LIBC_CHECK_CFG", "1"); - - if source_type == SourceType::InTree { - let mut lint_flags = Vec::new(); - // When extending this list, add the new lints to the RUSTFLAGS of the - // build_bootstrap function of src/bootstrap/bootstrap.py as well as - // some code doesn't go through this `rustc` wrapper. - lint_flags.push("-Wrust_2018_idioms"); - lint_flags.push("-Wunused_lifetimes"); - - if self.config.deny_warnings { - lint_flags.push("-Dwarnings"); - rustdocflags.arg("-Dwarnings"); - } - - // This does not use RUSTFLAGS due to caching issues with Cargo. - // Clippy is treated as an "in tree" tool, but shares the same - // cache as other "submodule" tools. With these options set in - // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. - // By injecting this into the rustc wrapper, this circumvents - // Cargo's fingerprint detection. This is fine because lint flags - // are always ignored in dependencies. Eventually this should be - // fixed via better support from Cargo. - cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); - - rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); - } - - if mode == Mode::Rustc { - rustflags.arg("-Wrustc::internal"); - // FIXME(edition_2024): Change this to `-Wrust_2024_idioms` when all - // of the individual lints are satisfied. - rustflags.arg("-Wkeyword_idents_2024"); - rustflags.arg("-Wunsafe_op_in_unsafe_fn"); - } - - if self.config.rust_frame_pointers { - rustflags.arg("-Cforce-frame-pointers=true"); - } - - // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc - // when compiling the standard library, since this might be linked into the final outputs - // produced by rustc. Since this mitigation is only available on Windows, only enable it - // for the standard library in case the compiler is run on a non-Windows platform. - // This is not needed for stage 0 artifacts because these will only be used for building - // the stage 1 compiler. - if cfg!(windows) - && mode == Mode::Std - && self.config.control_flow_guard - && compiler.stage >= 1 - { - rustflags.arg("-Ccontrol-flow-guard"); - } - - // If EHCont Guard is enabled, pass the `-Zehcont-guard` flag to rustc when compiling the - // standard library, since this might be linked into the final outputs produced by rustc. - // Since this mitigation is only available on Windows, only enable it for the standard - // library in case the compiler is run on a non-Windows platform. - // This is not needed for stage 0 artifacts because these will only be used for building - // the stage 1 compiler. - if cfg!(windows) && mode == Mode::Std && self.config.ehcont_guard && compiler.stage >= 1 { - rustflags.arg("-Zehcont-guard"); - } - - // For `cargo doc` invocations, make rustdoc print the Rust version into the docs - // This replaces spaces with tabs because RUSTDOCFLAGS does not - // support arguments with regular spaces. Hopefully someday Cargo will - // have space support. - let rust_version = self.rust_version().replace(' ', "\t"); - rustdocflags.arg("--crate-version").arg(&rust_version); - - // Environment variables *required* throughout the build - // - // FIXME: should update code to not require this env var - - // The host this new compiler will *run* on. - cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); - // The host this new compiler is being *built* on. - cargo.env("CFG_COMPILER_BUILD_TRIPLE", compiler.host.triple); - - // Set this for all builds to make sure doc builds also get it. - cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); - - // This one's a bit tricky. As of the time of this writing the compiler - // links to the `winapi` crate on crates.io. This crate provides raw - // bindings to Windows system functions, sort of like libc does for - // Unix. This crate also, however, provides "import libraries" for the - // MinGW targets. There's an import library per dll in the windows - // distribution which is what's linked to. These custom import libraries - // are used because the winapi crate can reference Windows functions not - // present in the MinGW import libraries. - // - // For example MinGW may ship libdbghelp.a, but it may not have - // references to all the functions in the dbghelp dll. Instead the - // custom import library for dbghelp in the winapi crates has all this - // information. - // - // Unfortunately for us though the import libraries are linked by - // default via `-ldylib=winapi_foo`. That is, they're linked with the - // `dylib` type with a `winapi_` prefix (so the winapi ones don't - // conflict with the system MinGW ones). This consequently means that - // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm - // DLL) when linked against *again*, for example with procedural macros - // or plugins, will trigger the propagation logic of `-ldylib`, passing - // `-lwinapi_foo` to the linker again. This isn't actually available in - // our distribution, however, so the link fails. - // - // To solve this problem we tell winapi to not use its bundled import - // libraries. This means that it will link to the system MinGW import - // libraries by default, and the `-ldylib=foo` directives will still get - // passed to the final linker, but they'll look like `-lfoo` which can - // be resolved because MinGW has the import library. The downside is we - // don't get newer functions from Windows, but we don't use any of them - // anyway. - if !mode.is_tool() { - cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); - } - - for _ in 0..self.verbosity { - cargo.arg("-v"); - } - - match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { - (Mode::Std, Some(n), _) | (_, _, Some(n)) => { - cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); - } - _ => { - // Don't set anything - } - } - - if self.config.locked_deps { - cargo.arg("--locked"); - } - if self.config.vendor || self.is_sudo { - cargo.arg("--frozen"); - } - - // Try to use a sysroot-relative bindir, in case it was configured absolutely. - cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); - - cargo.force_coloring_in_ci(); - - // When we build Rust dylibs they're all intended for intermediate - // usage, so make sure we pass the -Cprefer-dynamic flag instead of - // linking all deps statically into the dylib. - if matches!(mode, Mode::Std) { - rustflags.arg("-Cprefer-dynamic"); - } - if matches!(mode, Mode::Rustc) && !self.link_std_into_rustc_driver(target) { - rustflags.arg("-Cprefer-dynamic"); - } - - cargo.env( - "RUSTC_LINK_STD_INTO_RUSTC_DRIVER", - if self.link_std_into_rustc_driver(target) { "1" } else { "0" }, - ); - - // When building incrementally we default to a lower ThinLTO import limit - // (unless explicitly specified otherwise). This will produce a somewhat - // slower code but give way better compile times. - { - let limit = match self.config.rust_thin_lto_import_instr_limit { - Some(limit) => Some(limit), - None if self.config.incremental => Some(10), - _ => None, - }; - - if let Some(limit) = limit { - if stage == 0 - || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm" - { - rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); - } - } - } - - if matches!(mode, Mode::Std) { - if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { - rustflags.arg("-Zvalidate-mir"); - rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); - } - if self.config.rust_randomize_layout { - rustflags.arg("--cfg=randomized_layouts"); - } - // Always enable inlining MIR when building the standard library. - // Without this flag, MIR inlining is disabled when incremental compilation is enabled. - // That causes some mir-opt tests which inline functions from the standard library to - // break when incremental compilation is enabled. So this overrides the "no inlining - // during incremental builds" heuristic for the standard library. - rustflags.arg("-Zinline-mir"); - - // Similarly, we need to keep debug info for functions inlined into other std functions, - // even if we're not going to output debuginfo for the crate we're currently building, - // so that it'll be available when downstream consumers of std try to use it. - rustflags.arg("-Zinline-mir-preserve-debug"); - } - - Cargo { - command: cargo, - compiler, - target, - rustflags, - rustdocflags, - hostflags, - allow_features, - } - } -} diff --git a/standalonex/src/src/core/builder/mod.rs b/standalonex/src/src/core/builder/mod.rs deleted file mode 100644 index d59e0fa7..00000000 --- a/standalonex/src/src/core/builder/mod.rs +++ /dev/null @@ -1,1539 +0,0 @@ -mod cargo; - -use std::any::{Any, type_name}; -use std::cell::{Cell, RefCell}; -use std::collections::BTreeSet; -use std::fmt::{Debug, Write}; -use std::hash::Hash; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::sync::LazyLock; -use std::time::{Duration, Instant}; -use std::{env, fs}; - -use clap::ValueEnum; - -pub use self::cargo::Cargo; -pub use crate::Compiler; -use crate::core::build_steps::{ - check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, tool, vendor, -}; -use crate::core::config::flags::Subcommand; -use crate::core::config::{DryRun, TargetSelection}; -use crate::utils::cache::Cache; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t}; -use crate::{Build, Crate}; - -#[cfg(test)] -mod tests; - -/// Builds and performs different [`Self::kind`]s of stuff and actions, taking -/// into account build configuration from e.g. config.toml. -pub struct Builder<'a> { - /// Build configuration from e.g. config.toml. - pub build: &'a Build, - - /// The stage to use. Either implicitly determined based on subcommand, or - /// explicitly specified with `--stage N`. Normally this is the stage we - /// use, but sometimes we want to run steps with a lower stage than this. - pub top_stage: u32, - - /// What to build or what action to perform. - pub kind: Kind, - - /// A cache of outputs of [`Step`]s so we can avoid running steps we already - /// ran. - cache: Cache, - - /// A stack of [`Step`]s to run before we can run this builder. The output - /// of steps is cached in [`Self::cache`]. - stack: RefCell>>, - - /// The total amount of time we spent running [`Step`]s in [`Self::stack`]. - time_spent_on_dependencies: Cell, - - /// The paths passed on the command line. Used by steps to figure out what - /// to do. For example: with `./x check foo bar` we get `paths=["foo", - /// "bar"]`. - pub paths: Vec, -} - -impl Deref for Builder<'_> { - type Target = Build; - - fn deref(&self) -> &Self::Target { - self.build - } -} - -pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { - /// Result type of `Step::run`. - type Output: Clone; - - /// Whether this step is run by default as part of its respective phase, as defined by the `describe` - /// macro in [`Builder::get_step_descriptions`]. - /// - /// Note: Even if set to `true`, it can still be overridden with [`ShouldRun::default_condition`] - /// by `Step::should_run`. - const DEFAULT: bool = false; - - /// If true, then this rule should be skipped if --target was specified, but --host was not - const ONLY_HOSTS: bool = false; - - /// Primary function to implement `Step` logic. - /// - /// This function can be triggered in two ways: - /// 1. Directly from [`Builder::execute_cli`]. - /// 2. Indirectly by being called from other `Step`s using [`Builder::ensure`]. - /// - /// When called with [`Builder::execute_cli`] (as done by `Build::build`), this function executed twice: - /// - First in "dry-run" mode to validate certain things (like cyclic Step invocations, - /// directory creation, etc) super quickly. - /// - Then it's called again to run the actual, very expensive process. - /// - /// When triggered indirectly from other `Step`s, it may still run twice (as dry-run and real mode) - /// depending on the `Step::run` implementation of the caller. - fn run(self, builder: &Builder<'_>) -> Self::Output; - - /// Determines if this `Step` should be run when given specific paths (e.g., `x build $path`). - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; - - /// Called directly by the bootstrap `Step` handler when not triggered indirectly by other `Step`s using [`Builder::ensure`]. - /// For example, `./x.py test bootstrap` runs this for `test::Bootstrap`. Similarly, `./x.py test` runs it for every step - /// that is listed by the `describe` macro in [`Builder::get_step_descriptions`]. - fn make_run(_run: RunConfig<'_>) { - // It is reasonable to not have an implementation of make_run for rules - // who do not want to get called from the root context. This means that - // they are likely dependencies (e.g., sysroot creation) or similar, and - // as such calling them from ./x.py isn't logical. - unimplemented!() - } -} - -pub struct RunConfig<'a> { - pub builder: &'a Builder<'a>, - pub target: TargetSelection, - pub paths: Vec, -} - -impl RunConfig<'_> { - pub fn build_triple(&self) -> TargetSelection { - self.builder.build.build - } - - /// Return a list of crate names selected by `run.paths`. - #[track_caller] - pub fn cargo_crates_in_set(&self) -> Vec { - let mut crates = Vec::new(); - for krate in &self.paths { - let path = krate.assert_single_path(); - let Some(crate_name) = self.builder.crate_paths.get(&path.path) else { - panic!("missing crate for path {}", path.path.display()) - }; - crates.push(crate_name.to_string()); - } - crates - } - - /// Given an `alias` selected by the `Step` and the paths passed on the command line, - /// return a list of the crates that should be built. - /// - /// Normally, people will pass *just* `library` if they pass it. - /// But it's possible (although strange) to pass something like `library std core`. - /// Build all crates anyway, as if they hadn't passed the other args. - pub fn make_run_crates(&self, alias: Alias) -> Vec { - let has_alias = - self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); - if !has_alias { - return self.cargo_crates_in_set(); - } - - let crates = match alias { - Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), - Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), - }; - - crates.into_iter().map(|krate| krate.name.to_string()).collect() - } -} - -#[derive(Debug, Copy, Clone)] -pub enum Alias { - Library, - Compiler, -} - -impl Alias { - fn as_str(self) -> &'static str { - match self { - Alias::Library => "library", - Alias::Compiler => "compiler", - } - } -} - -/// A description of the crates in this set, suitable for passing to `builder.info`. -/// -/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. -pub fn crate_description(crates: &[impl AsRef]) -> String { - if crates.is_empty() { - return "".into(); - } - - let mut descr = String::from(" {"); - descr.push_str(crates[0].as_ref()); - for krate in &crates[1..] { - descr.push_str(", "); - descr.push_str(krate.as_ref()); - } - descr.push('}'); - descr -} - -struct StepDescription { - default: bool, - only_hosts: bool, - should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, - make_run: fn(RunConfig<'_>), - name: &'static str, - kind: Kind, -} - -#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] -pub struct TaskPath { - pub path: PathBuf, - pub kind: Option, -} - -impl Debug for TaskPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(kind) = &self.kind { - write!(f, "{}::", kind.as_str())?; - } - write!(f, "{}", self.path.display()) - } -} - -/// Collection of paths used to match a task rule. -#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] -pub enum PathSet { - /// A collection of individual paths or aliases. - /// - /// These are generally matched as a path suffix. For example, a - /// command-line value of `std` will match if `library/std` is in the - /// set. - /// - /// NOTE: the paths within a set should always be aliases of one another. - /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, - /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) - /// to build them separately. - Set(BTreeSet), - /// A "suite" of paths. - /// - /// These can match as a path suffix (like `Set`), or as a prefix. For - /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` - /// will match `tests/ui`. A command-line value of `ui` would also - /// match `tests/ui`. - Suite(TaskPath), -} - -impl PathSet { - fn empty() -> PathSet { - PathSet::Set(BTreeSet::new()) - } - - fn one>(path: P, kind: Kind) -> PathSet { - let mut set = BTreeSet::new(); - set.insert(TaskPath { path: path.into(), kind: Some(kind) }); - PathSet::Set(set) - } - - fn has(&self, needle: &Path, module: Kind) -> bool { - match self { - PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), - PathSet::Suite(suite) => Self::check(suite, needle, module), - } - } - - // internal use only - fn check(p: &TaskPath, needle: &Path, module: Kind) -> bool { - if let Some(p_kind) = &p.kind { - p.path.ends_with(needle) && *p_kind == module - } else { - p.path.ends_with(needle) - } - } - - /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the - /// matched needles. - /// - /// This is used for `StepDescription::krate`, which passes all matching crates at once to - /// `Step::make_run`, rather than calling it many times with a single crate. - /// See `tests.rs` for examples. - fn intersection_removing_matches(&self, needles: &mut Vec, module: Kind) -> PathSet { - let mut check = |p| { - for (i, n) in needles.iter().enumerate() { - let matched = Self::check(p, n, module); - if matched { - needles.remove(i); - return true; - } - } - false - }; - match self { - PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), - PathSet::Suite(suite) => { - if check(suite) { - self.clone() - } else { - PathSet::empty() - } - } - } - } - - /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. - /// - /// This can be used with [`ShouldRun::crate_or_deps`], [`ShouldRun::path`], or [`ShouldRun::alias`]. - #[track_caller] - pub fn assert_single_path(&self) -> &TaskPath { - match self { - PathSet::Set(set) => { - assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); - set.iter().next().unwrap() - } - PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), - } - } -} - -const PATH_REMAP: &[(&str, &[&str])] = &[ - // config.toml uses `rust-analyzer-proc-macro-srv`, but the - // actual path is `proc-macro-srv-cli` - ("rust-analyzer-proc-macro-srv", &["src/tools/rust-analyzer/crates/proc-macro-srv-cli"]), - // Make `x test tests` function the same as `x t tests/*` - ("tests", &[ - // tidy-alphabetical-start - "tests/assembly", - "tests/codegen", - "tests/codegen-units", - "tests/coverage", - "tests/coverage-run-rustdoc", - "tests/crashes", - "tests/debuginfo", - "tests/incremental", - "tests/mir-opt", - "tests/pretty", - "tests/run-make", - "tests/rustdoc", - "tests/rustdoc-gui", - "tests/rustdoc-js", - "tests/rustdoc-js-std", - "tests/rustdoc-json", - "tests/rustdoc-ui", - "tests/ui", - "tests/ui-fulldeps", - // tidy-alphabetical-end - ]), -]; - -fn remap_paths(paths: &mut Vec) { - let mut remove = vec![]; - let mut add = vec![]; - for (i, path) in paths.iter().enumerate().filter_map(|(i, path)| path.to_str().map(|s| (i, s))) - { - for &(search, replace) in PATH_REMAP { - // Remove leading and trailing slashes so `tests/` and `tests` are equivalent - if path.trim_matches(std::path::is_separator) == search { - remove.push(i); - add.extend(replace.iter().map(PathBuf::from)); - break; - } - } - } - remove.sort(); - remove.dedup(); - for idx in remove.into_iter().rev() { - paths.remove(idx); - } - paths.append(&mut add); -} - -impl StepDescription { - fn from(kind: Kind) -> StepDescription { - StepDescription { - default: S::DEFAULT, - only_hosts: S::ONLY_HOSTS, - should_run: S::should_run, - make_run: S::make_run, - name: std::any::type_name::(), - kind, - } - } - - fn maybe_run(&self, builder: &Builder<'_>, mut pathsets: Vec) { - pathsets.retain(|set| !self.is_excluded(builder, set)); - - if pathsets.is_empty() { - return; - } - - // Determine the targets participating in this rule. - let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; - - for target in targets { - let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; - (self.make_run)(run); - } - } - - fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { - if builder.config.skip.iter().any(|e| pathset.has(e, builder.kind)) { - if !matches!(builder.config.dry_run, DryRun::SelfCheck) { - println!("Skipping {pathset:?} because it is excluded"); - } - return true; - } - - if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { - builder.verbose(|| { - println!( - "{:?} not skipped for {:?} -- not in {:?}", - pathset, self.name, builder.config.skip - ) - }); - } - false - } - - fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { - let should_runs = v - .iter() - .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) - .collect::>(); - - if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install) - { - eprintln!( - "ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.", - builder.kind.as_str() - ); - crate::exit!(1); - } - - // sanity checks on rules - for (desc, should_run) in v.iter().zip(&should_runs) { - assert!( - !should_run.paths.is_empty(), - "{:?} should have at least one pathset", - desc.name - ); - } - - if paths.is_empty() || builder.config.include_default_paths { - for (desc, should_run) in v.iter().zip(&should_runs) { - if desc.default && should_run.is_really_default() { - desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); - } - } - } - - // Attempt to resolve paths to be relative to the builder source directory. - let mut paths: Vec = paths - .iter() - .map(|p| { - // If the path does not exist, it may represent the name of a Step, such as `tidy` in `x test tidy` - if !p.exists() { - return p.clone(); - } - - // Make the path absolute, strip the prefix, and convert to a PathBuf. - match std::path::absolute(p) { - Ok(p) => p.strip_prefix(&builder.src).unwrap_or(&p).to_path_buf(), - Err(e) => { - eprintln!("ERROR: {:?}", e); - panic!("Due to the above error, failed to resolve path: {:?}", p); - } - } - }) - .collect(); - - remap_paths(&mut paths); - - // Handle all test suite paths. - // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) - paths.retain(|path| { - for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(suite) = should_run.is_suite_path(path) { - desc.maybe_run(builder, vec![suite.clone()]); - return false; - } - } - true - }); - - if paths.is_empty() { - return; - } - - let mut path_lookup: Vec<(PathBuf, bool)> = - paths.clone().into_iter().map(|p| (p, false)).collect(); - - // List of `(usize, &StepDescription, Vec)` where `usize` is the closest index of a path - // compared to the given CLI paths. So we can respect to the CLI order by using this value to sort - // the steps. - let mut steps_to_run = vec![]; - - for (desc, should_run) in v.iter().zip(&should_runs) { - let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); - - // This value is used for sorting the step execution order. - // By default, `usize::MAX` is used as the index for steps to assign them the lowest priority. - // - // If we resolve the step's path from the given CLI input, this value will be updated with - // the step's actual index. - let mut closest_index = usize::MAX; - - // Find the closest index from the original list of paths given by the CLI input. - for (index, (path, is_used)) in path_lookup.iter_mut().enumerate() { - if !*is_used && !paths.contains(path) { - closest_index = index; - *is_used = true; - break; - } - } - - steps_to_run.push((closest_index, desc, pathsets)); - } - - // Sort the steps before running them to respect the CLI order. - steps_to_run.sort_by_key(|(index, _, _)| *index); - - // Handle all PathSets. - for (_index, desc, pathsets) in steps_to_run { - if !pathsets.is_empty() { - desc.maybe_run(builder, pathsets); - } - } - - if !paths.is_empty() { - eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); - eprintln!( - "HELP: run `x.py {} --help --verbose` to show a list of available paths", - builder.kind.as_str() - ); - eprintln!( - "NOTE: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" - ); - crate::exit!(1); - } - } -} - -enum ReallyDefault<'a> { - Bool(bool), - Lazy(LazyLock bool + 'a>>), -} - -pub struct ShouldRun<'a> { - pub builder: &'a Builder<'a>, - kind: Kind, - - // use a BTreeSet to maintain sort order - paths: BTreeSet, - - // If this is a default rule, this is an additional constraint placed on - // its run. Generally something like compiler docs being enabled. - is_really_default: ReallyDefault<'a>, -} - -impl<'a> ShouldRun<'a> { - fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { - ShouldRun { - builder, - kind, - paths: BTreeSet::new(), - is_really_default: ReallyDefault::Bool(true), // by default no additional conditions - } - } - - pub fn default_condition(mut self, cond: bool) -> Self { - self.is_really_default = ReallyDefault::Bool(cond); - self - } - - pub fn lazy_default_condition(mut self, lazy_cond: Box bool + 'a>) -> Self { - self.is_really_default = ReallyDefault::Lazy(LazyLock::new(lazy_cond)); - self - } - - pub fn is_really_default(&self) -> bool { - match &self.is_really_default { - ReallyDefault::Bool(val) => *val, - ReallyDefault::Lazy(lazy) => *lazy.deref(), - } - } - - /// Indicates it should run if the command-line selects the given crate or - /// any of its (local) dependencies. - /// - /// `make_run` will be called a single time with all matching command-line paths. - pub fn crate_or_deps(self, name: &str) -> Self { - let crates = self.builder.in_tree_crates(name, None); - self.crates(crates) - } - - /// Indicates it should run if the command-line selects any of the given crates. - /// - /// `make_run` will be called a single time with all matching command-line paths. - /// - /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible. - pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { - for krate in crates { - let path = krate.local_path(self.builder); - self.paths.insert(PathSet::one(path, self.kind)); - } - self - } - - // single alias, which does not correspond to any on-disk path - pub fn alias(mut self, alias: &str) -> Self { - // exceptional case for `Kind::Setup` because its `library` - // and `compiler` options would otherwise naively match with - // `compiler` and `library` folders respectively. - assert!( - self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), - "use `builder.path()` for real paths: {alias}" - ); - self.paths.insert(PathSet::Set( - std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), - )); - self - } - - // single, non-aliased path - pub fn path(self, path: &str) -> Self { - self.paths(&[path]) - } - - /// Multiple aliases for the same job. - /// - /// This differs from [`path`] in that multiple calls to path will end up calling `make_run` - /// multiple times, whereas a single call to `paths` will only ever generate a single call to - /// `paths`. - /// - /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible. - /// - /// [`path`]: ShouldRun::path - pub fn paths(mut self, paths: &[&str]) -> Self { - let submodules_paths = build_helper::util::parse_gitmodules(&self.builder.src); - - self.paths.insert(PathSet::Set( - paths - .iter() - .map(|p| { - // assert only if `p` isn't submodule - if !submodules_paths.iter().any(|sm_p| p.contains(sm_p)) { - assert!( - self.builder.src.join(p).exists(), - "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", - p - ); - } - - TaskPath { path: p.into(), kind: Some(self.kind) } - }) - .collect(), - )); - self - } - - /// Handles individual files (not directories) within a test suite. - fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { - self.paths.iter().find(|pathset| match pathset { - PathSet::Suite(suite) => requested_path.starts_with(&suite.path), - PathSet::Set(_) => false, - }) - } - - pub fn suite_path(mut self, suite: &str) -> Self { - self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); - self - } - - // allows being more explicit about why should_run in Step returns the value passed to it - pub fn never(mut self) -> ShouldRun<'a> { - self.paths.insert(PathSet::empty()); - self - } - - /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, - /// removing the matches from `paths`. - /// - /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work - /// within the same step. For example, `test::Crate` allows testing multiple crates in the same - /// cargo invocation, which are put into separate sets because they aren't aliases. - /// - /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing - /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) - fn pathset_for_paths_removing_matches( - &self, - paths: &mut Vec, - kind: Kind, - ) -> Vec { - let mut sets = vec![]; - for pathset in &self.paths { - let subset = pathset.intersection_removing_matches(paths, kind); - if subset != PathSet::empty() { - sets.push(subset); - } - } - sets - } -} - -#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, PartialOrd, Ord, ValueEnum)] -pub enum Kind { - #[value(alias = "b")] - Build, - #[value(alias = "c")] - Check, - Clippy, - Fix, - Format, - #[value(alias = "t")] - Test, - Miri, - MiriSetup, - MiriTest, - Bench, - #[value(alias = "d")] - Doc, - Clean, - Dist, - Install, - #[value(alias = "r")] - Run, - Setup, - Suggest, - Vendor, - Perf, -} - -impl Kind { - pub fn as_str(&self) -> &'static str { - match self { - Kind::Build => "build", - Kind::Check => "check", - Kind::Clippy => "clippy", - Kind::Fix => "fix", - Kind::Format => "fmt", - Kind::Test => "test", - Kind::Miri => "miri", - Kind::MiriSetup => panic!("`as_str` is not supported for `Kind::MiriSetup`."), - Kind::MiriTest => panic!("`as_str` is not supported for `Kind::MiriTest`."), - Kind::Bench => "bench", - Kind::Doc => "doc", - Kind::Clean => "clean", - Kind::Dist => "dist", - Kind::Install => "install", - Kind::Run => "run", - Kind::Setup => "setup", - Kind::Suggest => "suggest", - Kind::Vendor => "vendor", - Kind::Perf => "perf", - } - } - - pub fn description(&self) -> String { - match self { - Kind::Test => "Testing", - Kind::Bench => "Benchmarking", - Kind::Doc => "Documenting", - Kind::Run => "Running", - Kind::Suggest => "Suggesting", - Kind::Clippy => "Linting", - Kind::Perf => "Profiling & benchmarking", - _ => { - let title_letter = self.as_str()[0..1].to_ascii_uppercase(); - return format!("{title_letter}{}ing", &self.as_str()[1..]); - } - } - .to_owned() - } -} - -impl<'a> Builder<'a> { - fn get_step_descriptions(kind: Kind) -> Vec { - macro_rules! describe { - ($($rule:ty),+ $(,)?) => {{ - vec![$(StepDescription::from::<$rule>(kind)),+] - }}; - } - match kind { - Kind::Build => describe!( - compile::Std, - compile::Rustc, - compile::Assemble, - compile::CodegenBackend, - compile::StartupObjects, - tool::BuildManifest, - tool::Rustbook, - tool::ErrorIndex, - tool::UnstableBookGen, - tool::Tidy, - tool::Linkchecker, - tool::CargoTest, - tool::Compiletest, - tool::RemoteTestServer, - tool::RemoteTestClient, - tool::RustInstaller, - tool::Cargo, - tool::Rls, - tool::RustAnalyzer, - tool::RustAnalyzerProcMacroSrv, - tool::Rustdoc, - tool::Clippy, - tool::CargoClippy, - llvm::Llvm, - gcc::Gcc, - llvm::Sanitizers, - tool::Rustfmt, - tool::Miri, - tool::CargoMiri, - llvm::Lld, - llvm::Enzyme, - llvm::CrtBeginEnd, - tool::RustdocGUITest, - tool::OptimizedDist, - tool::CoverageDump, - tool::LlvmBitcodeLinker, - tool::RustcPerf, - ), - Kind::Clippy => describe!( - clippy::Std, - clippy::Rustc, - clippy::Bootstrap, - clippy::BuildHelper, - clippy::BuildManifest, - clippy::CargoMiri, - clippy::Clippy, - clippy::CollectLicenseMetadata, - clippy::Compiletest, - clippy::CoverageDump, - clippy::Jsondocck, - clippy::Jsondoclint, - clippy::LintDocs, - clippy::LlvmBitcodeLinker, - clippy::Miri, - clippy::MiroptTestTools, - clippy::OptDist, - clippy::RemoteTestClient, - clippy::RemoteTestServer, - clippy::Rls, - clippy::RustAnalyzer, - clippy::Rustdoc, - clippy::Rustfmt, - clippy::RustInstaller, - clippy::TestFloatParse, - clippy::Tidy, - clippy::CI, - ), - Kind::Check | Kind::Fix => describe!( - check::Std, - check::Rustc, - check::Rustdoc, - check::CodegenBackend, - check::Clippy, - check::Miri, - check::CargoMiri, - check::MiroptTestTools, - check::Rls, - check::Rustfmt, - check::RustAnalyzer, - check::TestFloatParse, - check::Bootstrap, - ), - Kind::Test => describe!( - crate::core::build_steps::toolstate::ToolStateCheck, - test::Tidy, - test::Ui, - test::Crashes, - test::Coverage, - test::CoverageMap, - test::CoverageRun, - test::MirOpt, - test::Codegen, - test::CodegenUnits, - test::Assembly, - test::Incremental, - test::Debuginfo, - test::UiFullDeps, - test::CodegenCranelift, - test::CodegenGCC, - test::Rustdoc, - test::CoverageRunRustdoc, - test::Pretty, - test::Crate, - test::CrateLibrustc, - test::CrateRustdoc, - test::CrateRustdocJsonTypes, - test::CrateBootstrap, - test::Linkcheck, - test::TierCheck, - test::Cargotest, - test::Cargo, - test::RustAnalyzer, - test::ErrorIndex, - test::Distcheck, - test::Nomicon, - test::Reference, - test::RustdocBook, - test::RustByExample, - test::TheBook, - test::UnstableBook, - test::RustcBook, - test::LintDocs, - test::RustcGuide, - test::EmbeddedBook, - test::EditionGuide, - test::Rustfmt, - test::Miri, - test::CargoMiri, - test::Clippy, - test::CompiletestTest, - test::CrateRunMakeSupport, - test::CrateBuildHelper, - test::RustdocJSStd, - test::RustdocJSNotStd, - test::RustdocGUI, - test::RustdocTheme, - test::RustdocUi, - test::RustdocJson, - test::HtmlCheck, - test::RustInstaller, - test::TestFloatParse, - // Run bootstrap close to the end as it's unlikely to fail - test::Bootstrap, - // Run run-make last, since these won't pass without make on Windows - test::RunMake, - ), - Kind::Miri => describe!(test::Crate), - Kind::Bench => describe!(test::Crate, test::CrateLibrustc), - Kind::Doc => describe!( - doc::UnstableBook, - doc::UnstableBookGen, - doc::TheBook, - doc::Standalone, - doc::Std, - doc::Rustc, - doc::Rustdoc, - doc::Rustfmt, - doc::ErrorIndex, - doc::Nomicon, - doc::Reference, - doc::RustdocBook, - doc::RustByExample, - doc::RustcBook, - doc::Cargo, - doc::CargoBook, - doc::Clippy, - doc::ClippyBook, - doc::Miri, - doc::EmbeddedBook, - doc::EditionGuide, - doc::StyleGuide, - doc::Tidy, - doc::Bootstrap, - doc::Releases, - doc::RunMakeSupport, - doc::BuildHelper, - doc::Compiletest, - ), - Kind::Dist => describe!( - dist::Docs, - dist::RustcDocs, - dist::JsonDocs, - dist::Mingw, - dist::Rustc, - dist::CodegenBackend, - dist::Std, - dist::RustcDev, - dist::Analysis, - dist::Src, - dist::Cargo, - dist::Rls, - dist::RustAnalyzer, - dist::Rustfmt, - dist::Clippy, - dist::Miri, - dist::LlvmTools, - dist::LlvmBitcodeLinker, - dist::RustDev, - dist::Bootstrap, - dist::Extended, - // It seems that PlainSourceTarball somehow changes how some of the tools - // perceive their dependencies (see #93033) which would invalidate fingerprints - // and force us to rebuild tools after vendoring dependencies. - // To work around this, create the Tarball after building all the tools. - dist::PlainSourceTarball, - dist::BuildManifest, - dist::ReproducibleArtifacts, - ), - Kind::Install => describe!( - install::Docs, - install::Std, - // During the Rust compiler (rustc) installation process, we copy the entire sysroot binary - // path (build/host/stage2/bin). Since the building tools also make their copy in the sysroot - // binary path, we must install rustc before the tools. Otherwise, the rust-installer will - // install the same binaries twice for each tool, leaving backup files (*.old) as a result. - install::Rustc, - install::Cargo, - install::RustAnalyzer, - install::Rustfmt, - install::Clippy, - install::Miri, - install::LlvmTools, - install::Src, - ), - Kind::Run => describe!( - run::BuildManifest, - run::BumpStage0, - run::ReplaceVersionPlaceholder, - run::Miri, - run::CollectLicenseMetadata, - run::GenerateCopyright, - run::GenerateWindowsSys, - run::GenerateCompletions, - run::UnicodeTableGenerator, - ), - Kind::Setup => { - describe!(setup::Profile, setup::Hook, setup::Link, setup::Editor) - } - Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), - Kind::Vendor => describe!(vendor::Vendor), - // special-cased in Build::build() - Kind::Format | Kind::Suggest | Kind::Perf => vec![], - Kind::MiriTest | Kind::MiriSetup => unreachable!(), - } - } - - pub fn get_help(build: &Build, kind: Kind) -> Option { - let step_descriptions = Builder::get_step_descriptions(kind); - if step_descriptions.is_empty() { - return None; - } - - let builder = Self::new_internal(build, kind, vec![]); - let builder = &builder; - // The "build" kind here is just a placeholder, it will be replaced with something else in - // the following statement. - let mut should_run = ShouldRun::new(builder, Kind::Build); - for desc in step_descriptions { - should_run.kind = desc.kind; - should_run = (desc.should_run)(should_run); - } - let mut help = String::from("Available paths:\n"); - let mut add_path = |path: &Path| { - t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); - }; - for pathset in should_run.paths { - match pathset { - PathSet::Set(set) => { - for path in set { - add_path(&path.path); - } - } - PathSet::Suite(path) => { - add_path(&path.path.join("...")); - } - } - } - Some(help) - } - - fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { - Builder { - build, - top_stage: build.config.stage, - kind, - cache: Cache::new(), - stack: RefCell::new(Vec::new()), - time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), - paths, - } - } - - pub fn new(build: &Build) -> Builder<'_> { - let paths = &build.config.paths; - let (kind, paths) = match build.config.cmd { - Subcommand::Build => (Kind::Build, &paths[..]), - Subcommand::Check { .. } => (Kind::Check, &paths[..]), - Subcommand::Clippy { .. } => (Kind::Clippy, &paths[..]), - Subcommand::Fix => (Kind::Fix, &paths[..]), - Subcommand::Doc { .. } => (Kind::Doc, &paths[..]), - Subcommand::Test { .. } => (Kind::Test, &paths[..]), - Subcommand::Miri { .. } => (Kind::Miri, &paths[..]), - Subcommand::Bench { .. } => (Kind::Bench, &paths[..]), - Subcommand::Dist => (Kind::Dist, &paths[..]), - Subcommand::Install => (Kind::Install, &paths[..]), - Subcommand::Run { .. } => (Kind::Run, &paths[..]), - Subcommand::Clean { .. } => (Kind::Clean, &paths[..]), - Subcommand::Format { .. } => (Kind::Format, &[][..]), - Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), - Subcommand::Setup { profile: ref path } => ( - Kind::Setup, - path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), - ), - Subcommand::Vendor { .. } => (Kind::Vendor, &paths[..]), - Subcommand::Perf { .. } => (Kind::Perf, &paths[..]), - }; - - Self::new_internal(build, kind, paths.to_owned()) - } - - pub fn execute_cli(&self) { - self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); - } - - pub fn default_doc(&self, paths: &[PathBuf]) { - self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); - } - - pub fn doc_rust_lang_org_channel(&self) -> String { - let channel = match &*self.config.channel { - "stable" => &self.version, - "beta" => "beta", - "nightly" | "dev" => "nightly", - // custom build of rustdoc maybe? link to the latest stable docs just in case - _ => "stable", - }; - - format!("https://doc.rust-lang.org/{channel}") - } - - fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { - StepDescription::run(v, self, paths); - } - - /// Returns if `std` should be statically linked into `rustc_driver`. - /// It's currently not done on `windows-gnu` due to linker bugs. - pub fn link_std_into_rustc_driver(&self, target: TargetSelection) -> bool { - !target.triple.ends_with("-windows-gnu") - } - - /// Obtain a compiler at a given stage and for a given host (i.e., this is the target that the - /// compiler will run on, *not* the target it will build code for). Explicitly does not take - /// `Compiler` since all `Compiler` instances are meant to be obtained through this function, - /// since it ensures that they are valid (i.e., built and assembled). - pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { - self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) - } - - /// Similar to `compiler`, except handles the full-bootstrap option to - /// silently use the stage1 compiler instead of a stage2 compiler if one is - /// requested. - /// - /// Note that this does *not* have the side effect of creating - /// `compiler(stage, host)`, unlike `compiler` above which does have such - /// a side effect. The returned compiler here can only be used to compile - /// new artifacts, it can't be used to rely on the presence of a particular - /// sysroot. - /// - /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. - pub fn compiler_for( - &self, - stage: u32, - host: TargetSelection, - target: TargetSelection, - ) -> Compiler { - if self.build.force_use_stage2(stage) { - self.compiler(2, self.config.build) - } else if self.build.force_use_stage1(stage, target) { - self.compiler(1, self.config.build) - } else { - self.compiler(stage, host) - } - } - - pub fn sysroot(&self, compiler: Compiler) -> PathBuf { - self.ensure(compile::Sysroot::new(compiler)) - } - - /// Returns the bindir for a compiler's sysroot. - pub fn sysroot_target_bindir(&self, compiler: Compiler, target: TargetSelection) -> PathBuf { - self.sysroot_target_libdir(compiler, target).parent().unwrap().join("bin") - } - - /// Returns the libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - pub fn sysroot_target_libdir(&self, compiler: Compiler, target: TargetSelection) -> PathBuf { - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - struct Libdir { - compiler: Compiler, - target: TargetSelection, - } - impl Step for Libdir { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder - .sysroot(self.compiler) - .join(lib) - .join("rustlib") - .join(self.target) - .join("lib"); - // Avoid deleting the rustlib/ directory we just copied - // (in `impl Step for Sysroot`). - if !builder.download_rustc() { - builder.verbose(|| { - println!("Removing sysroot {} to avoid caching bugs", sysroot.display()) - }); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - } - - if self.compiler.stage == 0 { - // The stage 0 compiler for the build triple is always pre-built. - // Ensure that `libLLVM.so` ends up in the target libdir, so that ui-fulldeps tests can use it when run. - dist::maybe_install_llvm_target( - builder, - self.compiler.host, - &builder.sysroot(self.compiler), - ); - } - - sysroot - } - } - self.ensure(Libdir { compiler, target }) - } - - pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { - self.sysroot_target_libdir(compiler, compiler.host).with_file_name("codegen-backends") - } - - /// Returns the compiler's libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `/lib` on Unix and `/bin` on - /// Windows. - pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.rustc_snapshot_libdir() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => { - self.sysroot(compiler).join(relative_libdir) - } - _ => self.sysroot(compiler).join(libdir(compiler.host)), - } - } - } - - /// Returns the compiler's relative libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `lib` on Unix and `bin` on - /// Windows. - pub fn libdir_relative(&self, compiler: Compiler) -> &Path { - if compiler.is_snapshot(self) { - libdir(self.config.build).as_ref() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ => libdir(compiler.host).as_ref(), - } - } - } - - /// Returns the compiler's relative libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - /// - /// For example this returns `lib` on Unix and Windows. - pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ if compiler.stage == 0 => &self.build.initial_libdir, - _ => Path::new("lib"), - } - } - - pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec { - let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; - - // Ensure that the downloaded LLVM libraries can be found. - if self.config.llvm_from_ci { - let ci_llvm_lib = self.out.join(compiler.host).join("ci-llvm").join("lib"); - dylib_dirs.push(ci_llvm_lib); - } - - dylib_dirs - } - - /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic - /// library lookup path. - pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut BootstrapCommand) { - // Windows doesn't need dylib path munging because the dlls for the - // compiler live next to the compiler and the system will find them - // automatically. - if cfg!(windows) { - return; - } - - add_dylib_path(self.rustc_lib_paths(compiler), cmd); - } - - /// Gets a path to the compiler specified. - pub fn rustc(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.initial_rustc.clone() - } else { - self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) - } - } - - /// Gets the paths to all of the compiler's codegen backends. - fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { - fs::read_dir(self.sysroot_codegen_backends(compiler)) - .into_iter() - .flatten() - .filter_map(Result::ok) - .map(|entry| entry.path()) - } - - pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { - self.ensure(tool::Rustdoc { compiler }) - } - - pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { - if run_compiler.stage == 0 { - let cargo_clippy = self - .config - .initial_cargo_clippy - .clone() - .unwrap_or_else(|| self.build.config.download_clippy()); - - let mut cmd = command(cargo_clippy); - cmd.env("CARGO", &self.initial_cargo); - return cmd; - } - - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - self.ensure(tool::Clippy { - compiler: build_compiler, - target: self.build.build, - extra_features: vec![], - }); - let cargo_clippy = self.ensure(tool::CargoClippy { - compiler: build_compiler, - target: self.build.build, - extra_features: vec![], - }); - let mut dylib_path = helpers::dylib_path(); - dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); - - let mut cmd = command(cargo_clippy); - cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - cmd.env("CARGO", &self.initial_cargo); - cmd - } - - pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { - assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0"); - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - - // Prepare the tools - let miri = self.ensure(tool::Miri { - compiler: build_compiler, - target: self.build.build, - extra_features: Vec::new(), - }); - let cargo_miri = self.ensure(tool::CargoMiri { - compiler: build_compiler, - target: self.build.build, - extra_features: Vec::new(), - }); - // Invoke cargo-miri, make sure it can find miri and cargo. - let mut cmd = command(cargo_miri); - cmd.env("MIRI", &miri); - cmd.env("CARGO", &self.initial_cargo); - // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, - // so they match the Miri we just built. However this means they are actually living one - // stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`. - // This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an - // "assemble" step like rustc does that would cross the stage boundary. We can't use - // `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to - // the PATH due to the stage mismatch. - // Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503. - add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd); - cmd - } - - pub fn rustdoc_cmd(&self, compiler: Compiler) -> BootstrapCommand { - let mut cmd = command(self.bootstrap_out.join("rustdoc")); - cmd.env("RUSTC_STAGE", compiler.stage.to_string()) - .env("RUSTC_SYSROOT", self.sysroot(compiler)) - // Note that this is *not* the sysroot_libdir because rustdoc must be linked - // equivalently to rustc. - .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) - .env("CFG_RELEASE_CHANNEL", &self.config.channel) - .env("RUSTDOC_REAL", self.rustdoc(compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - - cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); - - if self.config.deny_warnings { - cmd.arg("-Dwarnings"); - } - cmd.arg("-Znormalize-docs"); - cmd.args(linker_args(self, compiler.host, LldThreads::Yes)); - cmd - } - - /// Return the path to `llvm-config` for the target, if it exists. - /// - /// Note that this returns `None` if LLVM is disabled, or if we're in a - /// check build or dry-run, where there's no need to build all of LLVM. - fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() { - let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); - if llvm_config.is_file() { - return Some(llvm_config); - } - } - None - } - - /// Ensure that a given step is built, returning its output. This will - /// cache the step, so it is safe (and good!) to call this as often as - /// needed to ensure that all dependencies are built. - pub fn ensure(&'a self, step: S) -> S::Output { - { - let mut stack = self.stack.borrow_mut(); - for stack_step in stack.iter() { - // should skip - if stack_step.downcast_ref::().map_or(true, |stack_step| *stack_step != step) { - continue; - } - let mut out = String::new(); - out += &format!("\n\nCycle in build detected when adding {step:?}\n"); - for el in stack.iter().rev() { - out += &format!("\t{el:?}\n"); - } - panic!("{}", out); - } - if let Some(out) = self.cache.get(&step) { - self.verbose_than(1, || println!("{}c {:?}", " ".repeat(stack.len()), step)); - - return out; - } - self.verbose_than(1, || println!("{}> {:?}", " ".repeat(stack.len()), step)); - stack.push(Box::new(step.clone())); - } - - #[cfg(feature = "build-metrics")] - self.metrics.enter_step(&step, self); - - let (out, dur) = { - let start = Instant::now(); - let zero = Duration::new(0, 0); - let parent = self.time_spent_on_dependencies.replace(zero); - let out = step.clone().run(self); - let dur = start.elapsed(); - let deps = self.time_spent_on_dependencies.replace(parent + dur); - (out, dur - deps) - }; - - if self.config.print_step_timings && !self.config.dry_run() { - let step_string = format!("{step:?}"); - let brace_index = step_string.find('{').unwrap_or(0); - let type_string = type_name::(); - println!( - "[TIMING] {} {} -- {}.{:03}", - &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), - &step_string[brace_index..], - dur.as_secs(), - dur.subsec_millis() - ); - } - - #[cfg(feature = "build-metrics")] - self.metrics.exit_step(self); - - { - let mut stack = self.stack.borrow_mut(); - let cur_step = stack.pop().expect("step stack empty"); - assert_eq!(cur_step.downcast_ref(), Some(&step)); - } - self.verbose_than(1, || println!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); - self.cache.put(step, out.clone()); - out - } - - /// Ensure that a given step is built *only if it's supposed to be built by default*, returning - /// its output. This will cache the step, so it's safe (and good!) to call this as often as - /// needed to ensure that all dependencies are build. - pub(crate) fn ensure_if_default>>( - &'a self, - step: S, - kind: Kind, - ) -> S::Output { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - // Avoid running steps contained in --skip - for pathset in &should_run.paths { - if desc.is_excluded(self, pathset) { - return None; - } - } - - // Only execute if it's supposed to run as default - if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } - } - - /// Checks if any of the "should_run" paths is in the `Builder` paths. - pub(crate) fn was_invoked_explicitly(&'a self, kind: Kind) -> bool { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - for path in &self.paths { - if should_run.paths.iter().any(|s| s.has(path, desc.kind)) - && !desc.is_excluded( - self, - &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), - ) - { - return true; - } - } - - false - } - - pub(crate) fn maybe_open_in_browser(&self, path: impl AsRef) { - if self.was_invoked_explicitly::(Kind::Doc) { - self.open_in_browser(path); - } - } - - pub(crate) fn open_in_browser(&self, path: impl AsRef) { - if self.config.dry_run() || !self.config.cmd.open() { - return; - } - - let path = path.as_ref(); - self.info(&format!("Opening doc {}", path.display())); - if let Err(err) = opener::open(path) { - self.info(&format!("{err}\n")); - } - } -} diff --git a/standalonex/src/src/core/builder/tests.rs b/standalonex/src/src/core/builder/tests.rs deleted file mode 100644 index 21c5f723..00000000 --- a/standalonex/src/src/core/builder/tests.rs +++ /dev/null @@ -1,724 +0,0 @@ -use std::thread; - -use super::*; -use crate::Flags; -use crate::core::build_steps::doc::DocumentationFormat; -use crate::core::config::Config; - -fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { - configure_with_args(&[cmd.to_owned()], host, target) -} - -fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { - let mut config = Config::parse(Flags::parse(cmd)); - // don't save toolstates - config.save_toolstates = None; - config.dry_run = DryRun::SelfCheck; - - // Ignore most submodules, since we don't need them for a dry run, and the - // tests run much faster without them. - // - // The src/doc/book submodule is needed because TheBook step tries to - // access files even during a dry-run (may want to consider just skipping - // that in a dry run). - let submodule_build = Build::new(Config { - // don't include LLVM, so CI doesn't require ninja/cmake to be installed - rust_codegen_backends: vec![], - ..Config::parse(Flags::parse(&["check".to_owned()])) - }); - submodule_build.require_submodule("src/doc/book", None); - config.submodules = Some(false); - - config.ninja_in_file = false; - // try to avoid spurious failures in dist where we create/delete each others file - // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us - let dir = Path::new(env!("OUT_DIR")) - .join("tmp-rustbuild-tests") - .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); - t!(fs::create_dir_all(&dir)); - config.out = dir; - config.build = TargetSelection::from_user("A-A"); - config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); - config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); - config -} - -fn first(v: Vec<(A, B)>) -> Vec { - v.into_iter().map(|(a, _)| a).collect::>() -} - -fn run_build(paths: &[PathBuf], config: Config) -> Cache { - let kind = config.cmd.kind(); - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); - builder.cache -} - -fn check_cli(paths: [&str; N]) { - run_build( - &paths.map(PathBuf::from), - configure_with_args(&paths.map(String::from), &["A-A"], &["A-A"]), - ); -} - -macro_rules! std { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Std::new( - Compiler { - host: TargetSelection::from_user(concat!( - stringify!($host), - "-", - stringify!($host) - )), - stage: $stage, - }, - TargetSelection::from_user(concat!(stringify!($target), "-", stringify!($target))), - ) - }; -} - -macro_rules! doc_std { - ($host:ident => $target:ident, stage = $stage:literal) => {{ - doc::Std::new( - $stage, - TargetSelection::from_user(concat!(stringify!($target), "-", stringify!($target))), - DocumentationFormat::Html, - ) - }}; -} - -macro_rules! rustc { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Rustc::new( - Compiler { - host: TargetSelection::from_user(concat!( - stringify!($host), - "-", - stringify!($host) - )), - stage: $stage, - }, - TargetSelection::from_user(concat!(stringify!($target), "-", stringify!($target))), - ) - }; -} - -#[test] -fn test_valid() { - // make sure multi suite paths are accepted - check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); -} - -#[test] -#[should_panic] -fn test_invalid() { - // make sure that invalid paths are caught, even when combined with valid paths - check_cli(["test", "library/std", "x"]); -} - -#[test] -fn test_intersection() { - let set = |paths: &[&str]| { - PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect()) - }; - let library_set = set(&["library/core", "library/alloc", "library/std"]); - let mut command_paths = vec![ - PathBuf::from("library/core"), - PathBuf::from("library/alloc"), - PathBuf::from("library/stdarch"), - ]; - let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build); - assert_eq!(subset, set(&["library/core", "library/alloc"]),); - assert_eq!(command_paths, vec![PathBuf::from("library/stdarch")]); -} - -#[test] -fn validate_path_remap() { - let build = Build::new(configure("test", &["A-A"], &["A-A"])); - - PATH_REMAP - .iter() - .flat_map(|(_, paths)| paths.iter()) - .map(|path| build.src.join(path)) - .for_each(|path| { - assert!(path.exists(), "{} should exist.", path.display()); - }); -} - -#[test] -fn check_missing_paths_for_x_test_tests() { - let build = Build::new(configure("test", &["A-A"], &["A-A"])); - - let (_, tests_remap_paths) = - PATH_REMAP.iter().find(|(target_path, _)| *target_path == "tests").unwrap(); - - let tests_dir = fs::read_dir(build.src.join("tests")).unwrap(); - for dir in tests_dir { - let path = dir.unwrap().path(); - - // Skip if not a test directory. - if path.ends_with("tests/auxiliary") || !path.is_dir() { - continue; - } - - assert!( - tests_remap_paths.iter().any(|item| path.ends_with(*item)), - "{} is missing in PATH_REMAP tests list.", - path.display() - ); - } -} - -#[test] -fn test_exclude() { - let mut config = configure("test", &["A-A"], &["A-A"]); - config.skip = vec!["src/tools/tidy".into()]; - let cache = run_build(&[], config); - - // Ensure we have really excluded tidy - assert!(!cache.contains::()); - - // Ensure other tests are not affected. - assert!(cache.contains::()); -} - -#[test] -fn test_exclude_kind() { - let path = PathBuf::from("compiler/rustc_data_structures"); - - let mut config = configure("test", &["A-A"], &["A-A"]); - // Ensure our test is valid, and `test::Rustc` would be run without the exclude. - assert!(run_build(&[], config.clone()).contains::()); - // Ensure tests for rustc are not skipped. - config.skip = vec![path.clone()]; - assert!(run_build(&[], config.clone()).contains::()); - // Ensure builds for rustc are not skipped. - assert!(run_build(&[], config).contains::()); -} - -/// Ensure that if someone passes both a single crate and `library`, all library crates get built. -#[test] -fn alias_and_path_for_library() { - let mut cache = - run_build(&["library".into(), "core".into()], configure("build", &["A-A"], &["A-A"])); - assert_eq!(first(cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1) - ]); - - let mut cache = - run_build(&["library".into(), "core".into()], configure("doc", &["A-A"], &["A-A"])); - assert_eq!(first(cache.all::()), &[doc_std!(A => A, stage = 0)]); -} - -#[test] -fn ci_rustc_if_unchanged_logic() { - let config = Config::parse_inner( - Flags::parse(&[ - "build".to_owned(), - "--dry-run".to_owned(), - "--set=rust.download-rustc='if-unchanged'".to_owned(), - ]), - |&_| Ok(Default::default()), - ); - - let build = Build::new(config.clone()); - let builder = Builder::new(&build); - - if config.out.exists() { - fs::remove_dir_all(&config.out).unwrap(); - } - - builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]); - - // Make sure "if-unchanged" logic doesn't try to use CI rustc while there are changes - // in compiler and/or library. - if config.download_rustc_commit.is_some() { - let has_changes = - config.last_modified_commit(&["compiler", "library"], "download-rustc", true).is_none(); - - assert!( - !has_changes, - "CI-rustc can't be used with 'if-unchanged' while there are changes in compiler and/or library." - ); - } -} - -mod defaults { - use pretty_assertions::assert_eq; - - use super::{configure, first, run_build}; - use crate::Config; - use crate::core::builder::*; - - #[test] - fn build_default() { - let mut cache = run_build(&[], configure("build", &["A-A"], &["A-A"])); - - let a = TargetSelection::from_user("A-A"); - assert_eq!(first(cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - ]); - assert!(!cache.all::().is_empty()); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::()), - // Recall that rustdoc stages are off-by-one - // - this is the compiler it's _linked_ to, not built with. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], - ); - assert_eq!(first(cache.all::()), &[rustc!(A => A, stage = 0)],); - } - - #[test] - fn build_stage_0() { - let config = Config { stage: 0, ..configure("build", &["A-A"], &["A-A"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A-A"); - assert_eq!(first(cache.all::()), &[std!(A => A, stage = 0)]); - assert!(!cache.all::().is_empty()); - assert_eq!( - first(cache.all::()), - // This is the beta rustdoc. - // Add an assert here to make sure this is the only rustdoc built. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], - ); - assert!(cache.all::().is_empty()); - } - - #[test] - fn build_cross_compile() { - let config = Config { stage: 1, ..configure("build", &["A-A", "B-B"], &["A-A", "B-B"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A-A"); - let b = TargetSelection::from_user("B-B"); - - // Ideally, this build wouldn't actually have `target: a` - // rustdoc/rustcc/std here (the user only requested a host=B build, so - // there's not really a need for us to build for target A in this case - // (since we're producing stage 1 libraries/binaries). But currently - // bootstrap is just a bit buggy here; this should be fixed though. - assert_eq!(first(cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => B, stage = 0), - std!(A => B, stage = 1), - ]); - assert_eq!(first(cache.all::()), &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, - ]); - assert_eq!(first(cache.all::()), &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, - ],); - assert_eq!(first(cache.all::()), &[ - rustc!(A => A, stage = 0), - rustc!(A => B, stage = 0), - ]); - } - - #[test] - fn doc_default() { - let mut config = configure("doc", &["A-A"], &["A-A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let mut cache = run_build(&[], config); - let a = TargetSelection::from_user("A-A"); - - // error_index_generator uses stage 0 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!(first(cache.all::()), &[doc::ErrorIndex { target: a },]); - assert_eq!(first(cache.all::()), &[tool::ErrorIndex { - compiler: Compiler { host: a, stage: 0 } - }]); - // docs should be built with the beta compiler, not with the stage0 artifacts. - // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, - // not the one it was built by. - assert_eq!(first(cache.all::()), &[tool::Rustdoc { - compiler: Compiler { host: a, stage: 0 } - },]); - } -} - -mod dist { - use pretty_assertions::assert_eq; - - use super::{Config, first, run_build}; - use crate::core::builder::*; - - fn configure(host: &[&str], target: &[&str]) -> Config { - Config { stage: 2, ..super::configure("dist", host, target) } - } - - #[test] - fn dist_baseline() { - let mut cache = run_build(&[], configure(&["A-A"], &["A-A"])); - - let a = TargetSelection::from_user("A-A"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a },]); - assert_eq!(first(cache.all::()), &[dist::Rustc { - compiler: Compiler { host: a, stage: 2 } - },]); - assert_eq!(first(cache.all::()), &[dist::Std { - compiler: Compiler { host: a, stage: 1 }, - target: a - },]); - assert_eq!(first(cache.all::()), &[dist::Src]); - // Make sure rustdoc is only built once. - assert_eq!(first(cache.all::()), &[tool::Rustdoc { - compiler: Compiler { host: a, stage: 2 } - },]); - } - - #[test] - fn dist_with_targets() { - let mut cache = run_build(&[], configure(&["A-A"], &["A-A", "B-B"])); - - let a = TargetSelection::from_user("A-A"); - let b = TargetSelection::from_user("B-B"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a }, dist::Docs { - host: b - },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a }, dist::Mingw { - host: b - },]); - assert_eq!(first(cache.all::()), &[dist::Rustc { - compiler: Compiler { host: a, stage: 2 } - },]); - assert_eq!(first(cache.all::()), &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - ]); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_hosts() { - let mut cache = run_build(&[], configure(&["A-A", "B-B"], &["A-A", "B-B"])); - - let a = TargetSelection::from_user("A-A"); - let b = TargetSelection::from_user("B-B"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a }, dist::Docs { - host: b - },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a }, dist::Mingw { - host: b - },]); - assert_eq!(first(cache.all::()), &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ]); - assert_eq!(first(cache.all::()), &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ]); - assert_eq!(first(cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ],); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_only_cross_host() { - let b = TargetSelection::from_user("B-B"); - let mut config = configure(&["A-A", "B-B"], &["A-A", "B-B"]); - config.docs = false; - config.extended = true; - config.hosts = vec![b]; - let mut cache = run_build(&[], config); - - assert_eq!(first(cache.all::()), &[dist::Rustc { - compiler: Compiler { host: b, stage: 2 } - },]); - assert_eq!(first(cache.all::()), &[ - rustc!(A => A, stage = 0), - rustc!(A => B, stage = 1), - ]); - } - - #[test] - fn dist_with_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A-A", "B-B"], &["A-A", "B-B", "C-C"])); - - let a = TargetSelection::from_user("A-A"); - let b = TargetSelection::from_user("B-B"); - let c = TargetSelection::from_user("C-C"); - - assert_eq!(first(cache.all::()), &[ - dist::Docs { host: a }, - dist::Docs { host: b }, - dist::Docs { host: c }, - ]); - assert_eq!(first(cache.all::()), &[ - dist::Mingw { host: a }, - dist::Mingw { host: b }, - dist::Mingw { host: c }, - ]); - assert_eq!(first(cache.all::()), &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ]); - assert_eq!(first(cache.all::()), &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ]); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_empty_host() { - let config = configure(&[], &["C-C"]); - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A-A"); - let c = TargetSelection::from_user("C-C"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: c },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: c },]); - assert_eq!(first(cache.all::()), &[dist::Std { - compiler: Compiler { host: a, stage: 2 }, - target: c - },]); - } - - #[test] - fn dist_with_same_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A-A", "B-B"], &["A-A", "B-B"])); - - let a = TargetSelection::from_user("A-A"); - let b = TargetSelection::from_user("B-B"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a }, dist::Docs { - host: b - },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a }, dist::Mingw { - host: b - },]); - assert_eq!(first(cache.all::()), &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ]); - assert_eq!(first(cache.all::()), &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ]); - assert_eq!(first(cache.all::()), &[dist::Src]); - assert_eq!(first(cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ]); - assert_eq!(first(cache.all::()), &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, - ]); - } - - #[test] - fn build_all() { - let build = Build::new(configure(&["A-A", "B-B"], &["A-A", "B-B", "C-C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[ - "compiler/rustc".into(), - "library".into(), - ]); - - assert_eq!(first(builder.cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - std!(A => C, stage = 2), - ]); - assert_eq!(builder.cache.all::().len(), 5); - assert_eq!(first(builder.cache.all::()), &[ - rustc!(A => A, stage = 0), - rustc!(A => A, stage = 1), - rustc!(A => A, stage = 2), - rustc!(A => B, stage = 1), - rustc!(A => B, stage = 2), - ]); - } - - #[test] - fn llvm_out_behaviour() { - let mut config = configure(&["A-A"], &["B-B"]); - config.llvm_from_ci = true; - let build = Build::new(config.clone()); - - let target = TargetSelection::from_user("A-A"); - assert!(build.llvm_out(target).ends_with("ci-llvm")); - let target = TargetSelection::from_user("B-B"); - assert!(build.llvm_out(target).ends_with("llvm")); - - config.llvm_from_ci = false; - let build = Build::new(config.clone()); - let target = TargetSelection::from_user("A-A"); - assert!(build.llvm_out(target).ends_with("llvm")); - } - - #[test] - fn build_with_empty_host() { - let config = configure(&[], &["C-C"]); - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user("A-A"); - - assert_eq!(first(builder.cache.all::()), &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => C, stage = 2), - ]); - assert_eq!(first(builder.cache.all::()), &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - ]); - assert_eq!(first(builder.cache.all::()), &[ - rustc!(A => A, stage = 0), - rustc!(A => A, stage = 1), - ]); - } - - #[test] - fn test_with_no_doc_stage0() { - let mut config = configure(&["A-A"], &["A-A"]); - config.stage = 0; - config.paths = vec!["library/std".into()]; - config.cmd = Subcommand::Test { - test_args: vec![], - compiletest_rustc_args: vec![], - no_fail_fast: false, - no_doc: true, - doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - }; - - let build = Build::new(config); - let mut builder = Builder::new(&build); - - let host = TargetSelection::from_user("A-A"); - - builder.run_step_descriptions(&[StepDescription::from::(Kind::Test)], &[ - "library/std".into(), - ]); - - // Ensure we don't build any compiler artifacts. - assert!(!builder.cache.contains::()); - assert_eq!(first(builder.cache.all::()), &[test::Crate { - compiler: Compiler { host, stage: 0 }, - target: host, - mode: crate::Mode::Std, - crates: vec!["std".to_owned()], - },]); - } - - #[test] - fn doc_ci() { - let mut config = configure(&["A-A"], &["A-A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); - let a = TargetSelection::from_user("A-A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!(first(builder.cache.all::()), &[tool::ErrorIndex { - compiler: Compiler { host: a, stage: 1 } - }]); - // This is actually stage 1, but Rustdoc::run swaps out the compiler with - // stage minus 1 if --stage is not 0. Very confusing! - assert_eq!(first(builder.cache.all::()), &[tool::Rustdoc { - compiler: Compiler { host: a, stage: 2 } - },]); - } - - #[test] - fn test_docs() { - // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&["A-A"], &["A-A"]); - config.cmd = Subcommand::Test { - test_args: vec![], - compiletest_rustc_args: vec![], - no_fail_fast: false, - doc: true, - no_doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - }; - // Make sure rustfmt binary not being found isn't an error. - config.channel = "beta".to_string(); - let build = Build::new(config); - let mut builder = Builder::new(&build); - - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - let a = TargetSelection::from_user("A-A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!(first(builder.cache.all::()), &[tool::ErrorIndex { - compiler: Compiler { host: a, stage: 1 } - }]); - // Unfortunately rustdoc is built twice. Once from stage1 for compiletest - // (and other things), and once from stage0 for std crates. Ideally it - // would only be built once. If someone wants to fix this, it might be - // worth investigating if it would be possible to test std from stage1. - // Note that the stages here are +1 than what they actually are because - // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is - // not 0. - // - // The stage 0 copy is the one downloaded for bootstrapping. It is - // (currently) needed to run "cargo test" on the linkchecker, and - // should be relatively "free". - assert_eq!(first(builder.cache.all::()), &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ]); - } -} diff --git a/standalonex/src/src/core/config/flags.rs b/standalonex/src/src/core/config/flags.rs deleted file mode 100644 index bfeb8115..00000000 --- a/standalonex/src/src/core/config/flags.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! Command-line interface of the bootstrap build system. -//! -//! This module implements the command-line parsing of the build system which -//! has various flags to configure how it's run. - -use std::path::{Path, PathBuf}; - -use clap::{CommandFactory, Parser, ValueEnum}; - -use crate::core::build_steps::setup::Profile; -use crate::core::builder::{Builder, Kind}; -use crate::core::config::{Config, TargetSelectionList, target_selection_list}; -use crate::{Build, DocTests}; - -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Color { - Always, - Never, - #[default] - Auto, -} - -/// Whether to deny warnings, emit them as warnings, or use the default behavior -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Warnings { - Deny, - Warn, - #[default] - Default, -} - -/// Deserialized version of all flags for this compile. -#[derive(Debug, Parser)] -#[command( - override_usage = "x.py [options] [...]", - disable_help_subcommand(true), - about = "", - next_line_help(false) -)] -pub struct Flags { - #[command(subcommand)] - pub cmd: Subcommand, - - #[arg(global = true, short, long, action = clap::ArgAction::Count)] - /// use verbose output (-vv for very verbose) - pub verbose: u8, // each extra -v after the first is passed to Cargo - #[arg(global = true, short, long)] - /// use incremental compilation - pub incremental: bool, - #[arg(global = true, long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] - /// TOML configuration file for build - pub config: Option, - #[arg(global = true, long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// Build directory, overrides `build.build-dir` in `config.toml` - pub build_dir: Option, - - #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] - /// build target of the stage0 compiler - pub build: Option, - - #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] - /// host targets to build - pub host: Option, - - #[arg(global = true, long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] - /// target targets to build - pub target: Option, - - #[arg(global = true, long, value_name = "PATH")] - /// build paths to exclude - pub exclude: Vec, // keeping for client backward compatibility - #[arg(global = true, long, value_name = "PATH")] - /// build paths to skip - pub skip: Vec, - #[arg(global = true, long)] - /// include default paths in addition to the provided ones - pub include_default_paths: bool, - - #[arg(global = true, value_hint = clap::ValueHint::Other, long)] - pub rustc_error_format: Option, - - #[arg(global = true, long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] - /// command to run on failure - pub on_fail: Option, - #[arg(global = true, long)] - /// dry run; don't build anything - pub dry_run: bool, - /// Indicates whether to dump the work done from bootstrap shims - #[arg(global = true, long)] - pub dump_bootstrap_shims: bool, - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the - /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) - pub stage: Option, - - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage: Vec, - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) of the standard library to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage_std: Vec, - #[arg(global = true, long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// path to the root of the rust checkout - pub src: Option, - - #[arg( - global = true, - short, - long, - value_hint = clap::ValueHint::Other, - value_name = "JOBS" - )] - /// number of jobs to run in parallel - pub jobs: Option, - // This overrides the deny-warnings configuration option, - // which passes -Dwarnings to the compiler invocations. - #[arg(global = true, long)] - #[arg(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] - /// if value is deny, will deny warnings - /// if value is warn, will emit warnings - /// otherwise, use the default configured behaviour - pub warnings: Warnings, - - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] - /// rustc error format - pub error_format: Option, - #[arg(global = true, long)] - /// use message-format=json - pub json_output: bool, - - #[arg(global = true, long, value_name = "STYLE")] - #[arg(value_enum, default_value_t = Color::Auto)] - /// whether to use color in cargo and rustc output - pub color: Color, - - #[arg(global = true, long)] - /// Bootstrap uses this value to decide whether it should bypass locking the build process. - /// This is rarely needed (e.g., compiling the std library for different targets in parallel). - /// - /// Unless you know exactly what you are doing, you probably don't need this. - pub bypass_bootstrap_lock: bool, - - /// generate PGO profile with rustc build - #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_generate: Option, - /// use PGO profile for rustc build - #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_use: Option, - /// use PGO profile for LLVM build - #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub llvm_profile_use: Option, - // LLVM doesn't support a custom location for generating profile - // information. - // - // llvm_out/build/profiles/ is the location this writes to. - /// generate PGO profile with llvm built for rustc - #[arg(global = true, long)] - pub llvm_profile_generate: bool, - /// Enable BOLT link flags - #[arg(global = true, long)] - pub enable_bolt_settings: bool, - /// Skip stage0 compiler validation - #[arg(global = true, long)] - pub skip_stage0_validation: bool, - /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. - #[arg(global = true, long)] - pub reproducible_artifact: Vec, - #[arg(global = true)] - /// paths for the subcommand - pub paths: Vec, - /// override options in config.toml - #[arg(global = true, value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] - pub set: Vec, - /// arguments passed to subcommands - #[arg(global = true, last(true), value_name = "ARGS")] - pub free_args: Vec, -} - -impl Flags { - /// Check if ` -h -v` was passed. - /// If yes, print the available paths and return `true`. - pub fn try_parse_verbose_help(args: &[String]) -> bool { - // We need to check for ` -h -v`, in which case we list the paths - #[derive(Parser)] - #[command(disable_help_flag(true))] - struct HelpVerboseOnly { - #[arg(short, long)] - help: bool, - #[arg(global = true, short, long, action = clap::ArgAction::Count)] - pub verbose: u8, - #[arg(value_enum)] - cmd: Kind, - } - if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = - HelpVerboseOnly::try_parse_from(normalize_args(args)) - { - println!("NOTE: updating submodules before printing available paths"); - let config = Config::parse(Self::parse(&[String::from("build")])); - let build = Build::new(config); - let paths = Builder::get_help(&build, subcommand); - if let Some(s) = paths { - println!("{s}"); - } else { - panic!("No paths available for subcommand `{}`", subcommand.as_str()); - } - true - } else { - false - } - } - - pub fn parse(args: &[String]) -> Self { - Flags::parse_from(normalize_args(args)) - } -} - -fn normalize_args(args: &[String]) -> Vec { - let first = String::from("x.py"); - let it = std::iter::once(first).chain(args.iter().cloned()); - it.collect() -} - -#[derive(Debug, Clone, Default, clap::Subcommand)] -pub enum Subcommand { - #[command(aliases = ["b"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example, for a quick build of a usable - compiler: - ./x.py build --stage 1 library/std - This will build a compiler and standard library from the local source code. - Once this is done, build/$ARCH/stage1 contains a usable compiler. - If no arguments are passed then the default artifacts for that stage are - compiled. For example: - ./x.py build --stage 0 - ./x.py build ")] - /// Compile either the compiler or libraries - #[default] - Build, - #[command(aliases = ["c"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example: - ./x.py check library/std - If no arguments are passed then many artifacts are checked.")] - /// Compile either the compiler or libraries, using cargo check - Check { - #[arg(long)] - /// Check all targets - all_targets: bool, - }, - /// Run Clippy (uses rustup/cargo-installed clippy binary) - #[command(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run clippy against. For example: - ./x.py clippy library/core - ./x.py clippy library/core library/proc_macro")] - Clippy { - #[arg(long)] - fix: bool, - #[arg(long, requires = "fix")] - allow_dirty: bool, - #[arg(long, requires = "fix")] - allow_staged: bool, - /// clippy lints to allow - #[arg(global = true, short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] - allow: Vec, - /// clippy lints to deny - #[arg(global = true, short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] - deny: Vec, - /// clippy lints to warn on - #[arg(global = true, short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] - warn: Vec, - /// clippy lints to forbid - #[arg(global = true, short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] - forbid: Vec, - }, - /// Run cargo fix - #[command(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run `cargo fix` against. For example: - ./x.py fix library/core - ./x.py fix library/core library/proc_macro")] - Fix, - #[command( - name = "fmt", - long_about = "\n - Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if - formatting is correct and fails if it is not. For example: - ./x.py fmt - ./x.py fmt --check" - )] - /// Run rustfmt - Format { - /// check formatting instead of applying - #[arg(long)] - check: bool, - - /// apply to all appropriate files, not just those that have been modified - #[arg(long)] - all: bool, - }, - #[command(aliases = ["d"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories of documentation - to build. For example: - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc src/doc/book library/std - ./x.py doc library/std --json - ./x.py doc library/std --open - If no arguments are passed then everything is documented: - ./x.py doc - ./x.py doc --stage 1")] - /// Build documentation - Doc { - #[arg(long)] - /// open the docs in a browser - open: bool, - #[arg(long)] - /// render the documentation in JSON format in addition to the usual HTML format - json: bool, - }, - #[command(aliases = ["t"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to test directories that - should be compiled and run. For example: - ./x.py test tests/ui - ./x.py test library/std --test-args hash_map - ./x.py test library/std --stage 0 --no-doc - ./x.py test tests/ui --bless - ./x.py test tests/ui --compare-mode next-solver - Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; - just like `build library/std --stage N` it tests the compiler produced by the previous - stage. - Execute tool tests with a tool name argument: - ./x.py test tidy - If no arguments are passed then the complete artifacts for that stage are - compiled and tested. - ./x.py test - ./x.py test --stage 1")] - /// Build and run some test suites - Test { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - /// extra options to pass the compiler when running compiletest tests - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - compiletest_rustc_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - #[arg(long)] - /// whether to automatically update stderr/stdout files - bless: bool, - #[arg(long)] - /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) - extra_checks: Option, - #[arg(long)] - /// rerun tests even if the inputs are unchanged - force_rerun: bool, - #[arg(long)] - /// only run tests that result has been changed - only_modified: bool, - #[arg(long, value_name = "COMPARE MODE")] - /// mode describing what file the actual ui output will be compared to - compare_mode: Option, - #[arg(long, value_name = "check | build | run")] - /// force {check,build,run}-pass tests to this mode. - pass: Option, - #[arg(long, value_name = "auto | always | never")] - /// whether to execute run-* tests - run: Option, - #[arg(long)] - /// enable this to generate a Rustfix coverage file, which is saved in - /// `//rustfix_missing_coverage.txt` - rustfix_coverage: bool, - }, - /// Build and run some test suites *in Miri* - Miri { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - }, - /// Build and run some benchmarks - Bench { - #[arg(long, allow_hyphen_values(true))] - test_args: Vec, - }, - /// Clean out build directories - Clean { - #[arg(long)] - /// Clean the entire build directory (not used by default) - all: bool, - #[arg(long, value_name = "N")] - /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. - stage: Option, - }, - /// Build distribution artifacts - Dist, - /// Install distribution artifacts - Install, - #[command(aliases = ["r"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to tools to build and run. For - example: - ./x.py run src/tools/bump-stage0 - At least a tool needs to be called.")] - /// Run tools contained in this repository - Run { - /// arguments for the tool - #[arg(long, allow_hyphen_values(true))] - args: Vec, - }, - /// Set up the environment for development - #[command(long_about = format!( - "\n -x.py setup creates a `config.toml` which changes the defaults for x.py itself, -as well as setting up a git pre-push hook, VS Code config and toolchain link. -Arguments: - This subcommand accepts a 'profile' to use for builds. For example: - ./x.py setup library - The profile is optional and you will be prompted interactively if it is not given. - The following profiles are available: -{} - To only set up the git hook, editor config or toolchain link, you may use - ./x.py setup hook - ./x.py setup editor - ./x.py setup link", Profile::all_for_help(" ").trim_end()))] - Setup { - /// Either the profile for `config.toml` or another setup action. - /// May be omitted to set up interactively - #[arg(value_name = "|hook|editor|link")] - profile: Option, - }, - /// Suggest a subset of tests to run, based on modified files - #[command(long_about = "\n")] - Suggest { - /// run suggested tests - #[arg(long)] - run: bool, - }, - /// Vendor dependencies - Vendor { - /// Additional `Cargo.toml` to sync and vendor - #[arg(long)] - sync: Vec, - /// Always include version in subdir name - #[arg(long)] - versioned_dirs: bool, - }, - /// Perform profiling and benchmarking of the compiler using the - /// `rustc-perf-wrapper` tool. - /// - /// You need to pass arguments after `--`, e.g.`x perf -- cachegrind`. - Perf {}, -} - -impl Subcommand { - pub fn kind(&self) -> Kind { - match self { - Subcommand::Bench { .. } => Kind::Bench, - Subcommand::Build { .. } => Kind::Build, - Subcommand::Check { .. } => Kind::Check, - Subcommand::Clippy { .. } => Kind::Clippy, - Subcommand::Doc { .. } => Kind::Doc, - Subcommand::Fix { .. } => Kind::Fix, - Subcommand::Format { .. } => Kind::Format, - Subcommand::Test { .. } => Kind::Test, - Subcommand::Miri { .. } => Kind::Miri, - Subcommand::Clean { .. } => Kind::Clean, - Subcommand::Dist { .. } => Kind::Dist, - Subcommand::Install { .. } => Kind::Install, - Subcommand::Run { .. } => Kind::Run, - Subcommand::Setup { .. } => Kind::Setup, - Subcommand::Suggest { .. } => Kind::Suggest, - Subcommand::Vendor { .. } => Kind::Vendor, - Subcommand::Perf { .. } => Kind::Perf, - } - } - - pub fn compiletest_rustc_args(&self) -> Vec<&str> { - match *self { - Subcommand::Test { ref compiletest_rustc_args, .. } => { - compiletest_rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - } - } - - pub fn fail_fast(&self) -> bool { - match *self { - Subcommand::Test { no_fail_fast, .. } | Subcommand::Miri { no_fail_fast, .. } => { - !no_fail_fast - } - _ => false, - } - } - - pub fn doc_tests(&self) -> DocTests { - match *self { - Subcommand::Test { doc, no_doc, .. } | Subcommand::Miri { no_doc, doc, .. } => { - if doc { - DocTests::Only - } else if no_doc { - DocTests::No - } else { - DocTests::Yes - } - } - _ => DocTests::Yes, - } - } - - pub fn bless(&self) -> bool { - match *self { - Subcommand::Test { bless, .. } => bless, - _ => false, - } - } - - pub fn extra_checks(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), - _ => None, - } - } - - pub fn only_modified(&self) -> bool { - match *self { - Subcommand::Test { only_modified, .. } => only_modified, - _ => false, - } - } - - pub fn force_rerun(&self) -> bool { - match *self { - Subcommand::Test { force_rerun, .. } => force_rerun, - _ => false, - } - } - - pub fn rustfix_coverage(&self) -> bool { - match *self { - Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, - _ => false, - } - } - - pub fn compare_mode(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn pass(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn run(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn open(&self) -> bool { - match *self { - Subcommand::Doc { open, .. } => open, - _ => false, - } - } - - pub fn json(&self) -> bool { - match *self { - Subcommand::Doc { json, .. } => json, - _ => false, - } - } - - pub fn vendor_versioned_dirs(&self) -> bool { - match *self { - Subcommand::Vendor { versioned_dirs, .. } => versioned_dirs, - _ => false, - } - } - - pub fn vendor_sync_args(&self) -> Vec { - match self { - Subcommand::Vendor { sync, .. } => sync.clone(), - _ => vec![], - } - } -} - -/// Returns the shell completion for a given shell, if the result differs from the current -/// content of `path`. If `path` does not exist, always returns `Some`. -pub fn get_completion(shell: G, path: &Path) -> Option { - let mut cmd = Flags::command(); - let current = if !path.exists() { - String::new() - } else { - std::fs::read_to_string(path).unwrap_or_else(|_| { - eprintln!("couldn't read {}", path.display()); - crate::exit!(1) - }) - }; - let mut buf = Vec::new(); - clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); - if buf == current.as_bytes() { - return None; - } - Some(String::from_utf8(buf).expect("completion script should be UTF-8")) -} diff --git a/standalonex/src/src/core/config/mod.rs b/standalonex/src/src/core/config/mod.rs deleted file mode 100644 index 9f09dd13..00000000 --- a/standalonex/src/src/core/config/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[allow(clippy::module_inception)] -mod config; -pub mod flags; -#[cfg(test)] -mod tests; - -pub use config::*; diff --git a/standalonex/src/src/core/download.rs b/standalonex/src/src/core/download.rs deleted file mode 100644 index 4ec5d70d..00000000 --- a/standalonex/src/src/core/download.rs +++ /dev/null @@ -1,874 +0,0 @@ -use std::env; -use std::ffi::OsString; -use std::fs::{self, File}; -use std::io::{BufRead, BufReader, BufWriter, ErrorKind, Write}; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::OnceLock; - -use build_helper::ci::CiEnv; -use xz2::bufread::XzDecoder; - -use crate::core::config::BUILDER_CONFIG_FILENAME; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::helpers::{check_run, exe, hex_encode, move_file, program_out_of_date}; -use crate::{Config, t}; - -static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock = OnceLock::new(); - -/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. -fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { - #[allow(deprecated)] - config.try_run(cmd) -} - -fn extract_curl_version(out: &[u8]) -> semver::Version { - let out = String::from_utf8_lossy(out); - // The output should look like this: "curl .. ..." - out.lines() - .next() - .and_then(|line| line.split(" ").nth(1)) - .and_then(|version| semver::Version::parse(version).ok()) - .unwrap_or(semver::Version::new(1, 0, 0)) -} - -fn curl_version() -> semver::Version { - let mut curl = Command::new("curl"); - curl.arg("-V"); - let Ok(out) = curl.output() else { return semver::Version::new(1, 0, 0) }; - let out = out.stdout; - extract_curl_version(&out) -} - -/// Generic helpers that are useful anywhere in bootstrap. -impl Config { - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub(crate) fn create(&self, path: &Path, s: &str) { - if self.dry_run() { - return; - } - t!(fs::write(path, s)); - } - - pub(crate) fn remove(&self, f: &Path) { - if self.dry_run() { - return; - } - fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); - } - - /// Create a temporary directory in `out` and return its path. - /// - /// NOTE: this temporary directory is shared between all steps; - /// if you need an empty directory, create a new subdirectory inside it. - pub(crate) fn tempdir(&self) -> PathBuf { - let tmp = self.out.join("tmp"); - t!(fs::create_dir_all(&tmp)); - tmp - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Returns false if do not execute at all, otherwise returns its - /// `status.success()`. - pub(crate) fn check_run(&self, cmd: &mut BootstrapCommand) -> bool { - if self.dry_run() && !cmd.run_always { - return true; - } - self.verbose(|| println!("running: {cmd:?}")); - check_run(cmd, self.is_verbose()) - } - - /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true - /// on NixOS - fn should_fix_bins_and_dylibs(&self) -> bool { - let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return false, - Ok(output) if !output.status.success() => return false, - Ok(output) => { - let mut os_name = output.stdout; - if os_name.last() == Some(&b'\n') { - os_name.pop(); - } - if os_name != b"Linux" { - return false; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`. - // NOTE: this intentionally comes after the Linux check: - // - patchelf only works with ELF files, so no need to run it on Mac or Windows - // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. - if let Some(explicit_value) = self.patch_binaries_for_nix { - return explicit_value; - } - - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - let is_nixos = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => false, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(os_release) => BufReader::new(os_release).lines().any(|l| { - let l = l.expect("reading /etc/os-release"); - matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"") - }), - }; - if !is_nixos { - let in_nix_shell = env::var("IN_NIX_SHELL"); - if let Ok(in_nix_shell) = in_nix_shell { - eprintln!( - "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \ - you may need to set `patch-binaries-for-nix=true` in config.toml" - ); - } - } - is_nixos - }); - if val { - eprintln!("INFO: You seem to be using Nix."); - } - val - } - - /// Modifies the interpreter section of 'fname' to fix the dynamic linker, - /// or the RPATH section, to fix the dynamic library search path - /// - /// This is only required on NixOS and uses the PatchELF utility to - /// change the interpreter/RPATH of ELF executables. - /// - /// Please see for more information - fn fix_bin_or_dylib(&self, fname: &Path) { - assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); - println!("attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceLock = OnceLock::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = self.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = try_run( - self, - Command::new("nix-build").args([ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ]), - ) - .is_ok(); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - patchelf.args(&[ - OsString::from("--add-rpath"), - OsString::from(t!(fs::canonicalize(nix_deps_dir)).join("lib")), - ]); - if !path_is_dylib(fname) { - // Finally, set the correct .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(["--set-interpreter", dynamic_linker.trim_end()]); - } - - let _ = try_run(self, patchelf.arg(fname)); - } - - fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { - self.verbose(|| println!("download {url}")); - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // While bootstrap itself only supports http and https downloads, downstream forks might - // need to download components from other protocols. The match allows them adding more - // protocols without worrying about merge conflicts if we change the HTTP implementation. - match url.split_once("://").map(|(proto, _)| proto) { - Some("http") | Some("https") => { - self.download_http_with_retries(&tempfile, url, help_on_error) - } - Some(other) => panic!("unsupported protocol {other} in {url}"), - None => panic!("no protocol in {url}"), - } - t!( - move_file(&tempfile, dest_path), - format!("failed to rename {tempfile:?} to {dest_path:?}") - ); - } - - fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { - println!("downloading {url}"); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - // options should be kept in sync with - // src/bootstrap/src/core/download.rs - // for consistency - let mut curl = command("curl"); - curl.args([ - // follow redirect - "--location", - // timeout if speed is < 10 bytes/sec for > 30 seconds - "--speed-time", - "30", - "--speed-limit", - "10", - // timeout if cannot connect within 30 seconds - "--connect-timeout", - "30", - // output file - "--output", - tempfile.to_str().unwrap(), - // if there is an error, don't restart the download, - // instead continue where it left off. - "--continue-at", - "-", - // retry up to 3 times. note that this means a maximum of 4 - // attempts will be made, since the first attempt isn't a *re*try. - "--retry", - "3", - // show errors, even if --silent is specified - "--show-error", - // set timestamp of downloaded file to that of the server - "--remote-time", - // fail on non-ok http status - "--fail", - ]); - // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. - if CiEnv::is_ci() { - curl.arg("--silent"); - } else { - curl.arg("--progress-bar"); - } - // --retry-all-errors was added in 7.71.0, don't use it if curl is old. - if curl_version() >= semver::Version::new(7, 71, 0) { - curl.arg("--retry-all-errors"); - } - curl.arg(url); - if !self.check_run(&mut curl) { - if self.build.contains("windows-msvc") { - eprintln!("Fallback to PowerShell"); - for _ in 0..3 { - if try_run(self, Command::new("PowerShell.exe").args([ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), - ), - ])).is_err() { - return; - } - eprintln!("\nspurious failure, trying again"); - } - } - if !help_on_error.is_empty() { - eprintln!("{help_on_error}"); - } - crate::exit!(1); - } - } - - fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { - eprintln!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball), format!("file {} not found", tarball.display())); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - - let is_ci_rustc = dst.ends_with("ci-rustc"); - let is_ci_llvm = dst.ends_with("ci-llvm"); - - // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding - // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow. - // Cache the entries when we extract it so we only have to read it once. - let mut recorded_entries = if is_ci_rustc { recorded_entries(dst, pattern) } else { None }; - - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - let is_builder_config = short_path.to_str() == Some(BUILDER_CONFIG_FILENAME); - - if !(short_path.starts_with(pattern) - || ((is_ci_rustc || is_ci_llvm) && is_builder_config)) - { - continue; - } - short_path = short_path.strip_prefix(pattern).unwrap_or(short_path); - let dst_path = dst.join(short_path); - self.verbose(|| { - println!("extracting {} to {}", original_path.display(), dst.display()) - }); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - if let Some(record) = &mut recorded_entries { - t!(writeln!(record, "{}", short_path.to_str().unwrap())); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(move_file(src_path, dst_path)); - } - let dst_dir = dst.join(directory_prefix); - if dst_dir.exists() { - t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display())); - } - } - - /// Returns whether the SHA256 checksum of `path` matches `expected`. - pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { - use sha2::Digest; - - self.verbose(|| println!("verifying {}", path.display())); - - if self.dry_run() { - return false; - } - - let mut hasher = sha2::Sha256::new(); - - let file = t!(File::open(path)); - let mut reader = BufReader::new(file); - - loop { - let buffer = t!(reader.fill_buf()); - let l = buffer.len(); - // break if EOF - if l == 0 { - break; - } - hasher.update(buffer); - reader.consume(l); - } - - let checksum = hex_encode(hasher.finalize().as_slice()); - let verified = checksum == expected; - - if !verified { - println!( - "invalid checksum: \n\ - found: {checksum}\n\ - expected: {expected}", - ); - } - - verified - } -} - -fn recorded_entries(dst: &Path, pattern: &str) -> Option> { - let name = if pattern == "rustc-dev" { - ".rustc-dev-contents" - } else if pattern.starts_with("rust-std") { - ".rust-std-contents" - } else { - return None; - }; - Some(BufWriter::new(t!(File::create(dst.join(name))))) -} - -enum DownloadSource { - CI, - Dist, -} - -/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. -impl Config { - pub(crate) fn download_clippy(&self) -> PathBuf { - self.verbose(|| println!("downloading stage0 clippy artifacts")); - - let date = &self.stage0_metadata.compiler.date; - let version = &self.stage0_metadata.compiler.version; - let host = self.build; - - let bin_root = self.out.join(host).join("stage0"); - let clippy_stamp = bin_root.join(".clippy-stamp"); - let cargo_clippy = bin_root.join("bin").join(exe("cargo-clippy", host)); - if cargo_clippy.exists() && !program_out_of_date(&clippy_stamp, date) { - return cargo_clippy; - } - - let filename = format!("clippy-{version}-{host}.tar.xz"); - self.download_component(DownloadSource::Dist, filename, "clippy-preview", date, "stage0"); - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&cargo_clippy); - self.fix_bin_or_dylib(&cargo_clippy.with_file_name(exe("clippy-driver", host))); - } - - self.create(&clippy_stamp, date); - cargo_clippy - } - - #[cfg(feature = "bootstrap-self-test")] - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - None - } - - /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't - /// reuse target directories or artifacts - #[cfg(not(feature = "bootstrap-self-test"))] - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - use build_helper::stage0_parser::VersionMetadata; - - let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; - let channel = format!("{version}-{date}"); - - let host = self.build; - let bin_root = self.out.join(host).join("rustfmt"); - let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); - if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { - return Some(rustfmt_path); - } - - self.download_component( - DownloadSource::Dist, - format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), - "rustfmt-preview", - date, - "rustfmt", - ); - self.download_component( - DownloadSource::Dist, - format!("rustc-{version}-{build}.tar.xz", build = host.triple), - "rustc", - date, - "rustfmt", - ); - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if path_is_dylib(&lib.path()) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - self.create(&rustfmt_stamp, &channel); - Some(rustfmt_path) - } - - pub(crate) fn ci_rust_std_contents(&self) -> Vec { - self.ci_component_contents(".rust-std-contents") - } - - pub(crate) fn ci_rustc_dev_contents(&self) -> Vec { - self.ci_component_contents(".rustc-dev-contents") - } - - fn ci_component_contents(&self, stamp_file: &str) -> Vec { - assert!(self.download_rustc()); - if self.dry_run() { - return vec![]; - } - - let ci_rustc_dir = self.ci_rustc_dir(); - let stamp_file = ci_rustc_dir.join(stamp_file); - let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); - t!(BufReader::new(contents_file).lines().collect()) - } - - pub(crate) fn download_ci_rustc(&self, commit: &str) { - self.verbose(|| println!("using downloaded stage2 artifacts from CI (commit {commit})")); - - let version = self.artifact_version_part(commit); - // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the - // `rustc_private` crates for tools. - let extra_components = ["rustc-dev"]; - - self.download_toolchain( - &version, - "ci-rustc", - &format!("{commit}-{}", self.llvm_assertions), - &extra_components, - Self::download_ci_component, - ); - } - - #[cfg(feature = "bootstrap-self-test")] - pub(crate) fn download_beta_toolchain(&self) {} - - #[cfg(not(feature = "bootstrap-self-test"))] - pub(crate) fn download_beta_toolchain(&self) { - self.verbose(|| println!("downloading stage0 beta artifacts")); - - let date = &self.stage0_metadata.compiler.date; - let version = &self.stage0_metadata.compiler.version; - let extra_components = ["cargo"]; - - let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { - config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") - }; - - self.download_toolchain( - version, - "stage0", - date, - &extra_components, - download_beta_component, - ); - } - - fn download_toolchain( - &self, - version: &str, - sysroot: &str, - stamp_key: &str, - extra_components: &[&str], - download_component: fn(&Config, String, &str, &str), - ) { - let host = self.build.triple; - let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = bin_root.join(".rustc-stamp"); - - if !bin_root.join("bin").join(exe("rustc", self.build)).exists() - || program_out_of_date(&rustc_stamp, stamp_key) - { - if bin_root.exists() { - t!(fs::remove_dir_all(&bin_root)); - } - let filename = format!("rust-std-{version}-{host}.tar.xz"); - let pattern = format!("rust-std-{host}"); - download_component(self, filename, &pattern, stamp_key); - let filename = format!("rustc-{version}-{host}.tar.xz"); - download_component(self, filename, "rustc", stamp_key); - - for component in extra_components { - let filename = format!("{component}-{version}-{host}.tar.xz"); - download_component(self, filename, component, stamp_key); - } - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); - self.fix_bin_or_dylib( - &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"), - ); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if path_is_dylib(&lib.path()) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - t!(fs::write(rustc_stamp, stamp_key)); - } - } - - /// Download a single component of a CI-built toolchain (not necessarily a published nightly). - // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident - fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { - Self::download_component( - self, - DownloadSource::CI, - filename, - prefix, - commit_with_assertions, - "ci-rustc", - ) - } - - fn download_component( - &self, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, - ) { - if self.dry_run() { - return; - } - - let cache_dst = - self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); - - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = self.out.join(self.build).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => { - let dist_server = if self.llvm_assertions { - self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() - } else { - self.stage0_metadata.config.artifacts_server.clone() - }; - let url = format!( - "{}/{filename}", - key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() - ); - (dist_server, url, false) - } - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0 - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - let checksum = if should_verify { - let error = format!( - "src/stage0 doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if self.verify(&tarball, sha256) { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - self.verbose(|| { - println!( - "ignoring cached file {} due to failed verification", - tarball.display() - ) - }); - self.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - let mut help_on_error = ""; - if destination == "ci-rustc" { - help_on_error = "ERROR: failed to download pre-built rustc from CI - -NOTE: old builds get deleted after a certain time -HELP: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: - -[rust] -download-rustc = false -"; - } - self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); - if let Some(sha256) = checksum { - if !self.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } - } - - self.unpack(&tarball, &bin_root, prefix); - } - - #[cfg(feature = "bootstrap-self-test")] - pub(crate) fn maybe_download_ci_llvm(&self) {} - - #[cfg(not(feature = "bootstrap-self-test"))] - pub(crate) fn maybe_download_ci_llvm(&self) { - use build_helper::exit; - - use crate::core::build_steps::llvm::detect_llvm_sha; - use crate::core::config::check_incompatible_options_for_ci_llvm; - - if !self.llvm_from_ci { - return; - } - - let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { - self.download_ci_llvm(&llvm_sha); - - if self.should_fix_bins_and_dylibs() { - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - self.fix_bin_or_dylib(&t!(entry).path()); - } - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = std::time::SystemTime::now(); - let file_times = fs::FileTimes::new().set_accessed(now).set_modified(now); - - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); - t!(crate::utils::helpers::set_file_times(llvm_config, file_times)); - - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(llvm_lib)) { - let lib = t!(entry).path(); - if path_is_dylib(&lib) { - self.fix_bin_or_dylib(&lib); - } - } - } - - t!(fs::write(llvm_stamp, key)); - } - - if let Some(config_path) = &self.config { - let current_config_toml = Self::get_toml(config_path).unwrap(); - - match self.get_builder_toml("ci-llvm") { - Ok(ci_config_toml) => { - t!(check_incompatible_options_for_ci_llvm(current_config_toml, ci_config_toml)); - } - Err(e) if e.to_string().contains("unknown field") => { - println!( - "WARNING: CI LLVM has some fields that are no longer supported in bootstrap; download-ci-llvm will be disabled." - ); - println!("HELP: Consider rebasing to a newer commit if available."); - } - Err(e) => { - eprintln!("ERROR: Failed to parse CI LLVM config.toml: {e}"); - exit!(2); - } - }; - }; - } - - #[cfg(not(feature = "bootstrap-self-test"))] - fn download_ci_llvm(&self, llvm_sha: &str) { - let llvm_assertions = self.llvm_assertions; - - let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); - let cache_dst = - self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); - - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &self.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &self.stage0_metadata.config.artifacts_server - }; - let version = self.artifact_version_part(llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "ERROR: failed to download llvm from ci - - HELP: There could be two reasons behind this: - 1) The host triple is not supported for `download-ci-llvm`. - 2) Old builds get deleted after a certain time. - HELP: In either case, disable `download-ci-llvm` in your config.toml: - - [llvm] - download-ci-llvm = false - "; - self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); - } - let llvm_root = self.ci_llvm_root(); - self.unpack(&tarball, &llvm_root, "rust-dev"); - } -} - -fn path_is_dylib(path: &Path) -> bool { - // The .so is not necessarily the extension, it might be libLLVM.so.18.1 - path.to_str().map_or(false, |path| path.contains(".so")) -} - -/// Checks whether the CI rustc is available for the given target triple. -pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool { - // All tier 1 targets and tier 2 targets with host tools. - const SUPPORTED_PLATFORMS: &[&str] = &[ - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "loongarch64-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - //"x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ]; - - const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] = - &["x86_64-pc-windows-msvc"]; - - if llvm_assertions { - SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple) - } else { - SUPPORTED_PLATFORMS.contains(&target_triple) - } -} diff --git a/standalonex/src/src/core/metadata.rs b/standalonex/src/src/core/metadata.rs deleted file mode 100644 index 983674d2..00000000 --- a/standalonex/src/src/core/metadata.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::collections::BTreeMap; -use std::path::PathBuf; - -use serde_derive::Deserialize; - -use crate::utils::exec::command; -use crate::{Build, Crate, t}; - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Output { - packages: Vec, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Package { - name: String, - source: Option, - manifest_path: String, - dependencies: Vec, - targets: Vec, - features: BTreeMap>, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Dependency { - name: String, - source: Option, -} - -#[derive(Debug, Deserialize)] -struct Target { - kind: Vec, -} - -/// Collects and stores package metadata of each workspace members into `build`, -/// by executing `cargo metadata` commands. -pub fn build(build: &mut Build) { - for package in workspace_members(build) { - if package.source.is_none() { - let name = package.name; - let mut path = PathBuf::from(package.manifest_path); - path.pop(); - let deps = package - .dependencies - .into_iter() - .filter(|dep| dep.source.is_none()) - .map(|dep| dep.name) - .collect(); - let has_lib = package.targets.iter().any(|t| t.kind.iter().any(|k| k == "lib")); - let krate = Crate { - name: name.clone(), - deps, - path, - has_lib, - features: package.features.keys().cloned().collect(), - }; - let relative_path = krate.local_path(build); - build.crates.insert(name.clone(), krate); - let existing_path = build.crate_paths.insert(relative_path, name); - assert!( - existing_path.is_none(), - "multiple crates with the same path: {}", - existing_path.unwrap() - ); - } - } -} - -/// Invokes `cargo metadata` to get package metadata of each workspace member. -/// -/// This is used to resolve specific crate paths in `fn should_run` to compile -/// particular crate (e.g., `x build sysroot` to build library/sysroot). -fn workspace_members(build: &Build) -> Vec { - let collect_metadata = |manifest_path| { - let mut cargo = command(&build.initial_cargo); - cargo - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("metadata") - .arg("--format-version") - .arg("1") - .arg("--no-deps") - .arg("--manifest-path") - .arg(build.src.join(manifest_path)); - let metadata_output = cargo.run_always().run_capture_stdout(build).stdout(); - let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); - packages - }; - - // Collects `metadata.packages` from the root and library workspaces. - let mut packages = vec![]; - packages.extend(collect_metadata("Cargo.toml")); - packages.extend(collect_metadata("library/Cargo.toml")); - packages -} diff --git a/standalonex/src/src/core/mod.rs b/standalonex/src/src/core/mod.rs deleted file mode 100644 index 9e18d670..00000000 --- a/standalonex/src/src/core/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub(crate) mod build_steps; -pub(crate) mod builder; -pub(crate) mod config; -pub(crate) mod download; -pub(crate) mod metadata; -pub(crate) mod sanity; diff --git a/standalonex/src/src/core/sanity.rs b/standalonex/src/src/core/sanity.rs deleted file mode 100644 index fcc9ea86..00000000 --- a/standalonex/src/src/core/sanity.rs +++ /dev/null @@ -1,388 +0,0 @@ -//! Sanity checking performed by bootstrap before actually executing anything. -//! -//! This module contains the implementation of ensuring that the build -//! environment looks reasonable before progressing. This will verify that -//! various programs like git and python exist, along with ensuring that all C -//! compilers for cross-compiling are found. -//! -//! In theory if we get past this phase it's a bug if a build fails, but in -//! practice that's likely not true! - -use std::collections::{HashMap, HashSet}; -use std::ffi::{OsStr, OsString}; -use std::path::PathBuf; -use std::{env, fs}; - -use crate::Build; -#[cfg(not(feature = "bootstrap-self-test"))] -use crate::builder::Builder; -use crate::builder::Kind; -#[cfg(not(feature = "bootstrap-self-test"))] -use crate::core::build_steps::tool; -use crate::core::config::Target; -use crate::utils::exec::command; - -pub struct Finder { - cache: HashMap>, - path: OsString, -} - -// During sanity checks, we search for target names to determine if they exist in the compiler's built-in -// target list (`rustc --print target-list`). While a target name may be present in the stage2 compiler, -// it might not yet be included in stage0. In such cases, we handle the targets missing from stage0 in this list. -// -// Targets can be removed from this list once they are present in the stage0 compiler (usually by updating the beta compiler of the bootstrap). -const STAGE0_MISSING_TARGETS: &[&str] = &[ - // just a dummy comment so the list doesn't get onelined - "sbf-solana-solana", - "sbpf-solana-solana", - "sbpfv0-solana-solana", - "sbpfv1-solana-solana", - "sbpfv2-solana-solana", - "sbpfv3-solana-solana", - "sbpfv4-solana-solana", -]; - -/// Minimum version threshold for libstdc++ required when using prebuilt LLVM -/// from CI (with`llvm.download-ci-llvm` option). -#[cfg(not(feature = "bootstrap-self-test"))] -const LIBSTDCXX_MIN_VERSION_THRESHOLD: usize = 8; - -impl Finder { - pub fn new() -> Self { - Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } - } - - pub fn maybe_have>(&mut self, cmd: S) -> Option { - let cmd: OsString = cmd.into(); - let path = &self.path; - self.cache - .entry(cmd.clone()) - .or_insert_with(|| { - for path in env::split_paths(path) { - let target = path.join(&cmd); - let mut cmd_exe = cmd.clone(); - cmd_exe.push(".exe"); - - if target.is_file() // some/path/git - || path.join(&cmd_exe).exists() // some/path/git.exe - || target.join(&cmd_exe).exists() - // some/path/git/git.exe - { - return Some(target); - } - } - None - }) - .clone() - } - - pub fn must_have>(&mut self, cmd: S) -> PathBuf { - self.maybe_have(&cmd).unwrap_or_else(|| { - panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); - }) - } -} - -pub fn check(build: &mut Build) { - let mut skip_target_sanity = - env::var_os("BOOTSTRAP_SKIP_TARGET_SANITY").is_some_and(|s| s == "1" || s == "true"); - - skip_target_sanity |= build.config.cmd.kind() == Kind::Check; - - // Skip target sanity checks when we are doing anything with mir-opt tests or Miri - let skipped_paths = [OsStr::new("mir-opt"), OsStr::new("miri")]; - skip_target_sanity |= build.config.paths.iter().any(|path| { - path.components().any(|component| skipped_paths.contains(&component.as_os_str())) - }); - - let path = env::var_os("PATH").unwrap_or_default(); - // On Windows, quotes are invalid characters for filename paths, and if - // one is present as part of the PATH then that can lead to the system - // being unable to identify the files properly. See - // https://github.com/rust-lang/rust/issues/34959 for more details. - if cfg!(windows) && path.to_string_lossy().contains('\"') { - panic!("PATH contains invalid character '\"'"); - } - - let mut cmd_finder = Finder::new(); - // If we've got a git directory we're gonna need git to update - // submodules and learn about various other aspects. - if build.rust_info().is_managed_git_subrepository() { - cmd_finder.must_have("git"); - } - - // Ensure that a compatible version of libstdc++ is available on the system when using `llvm.download-ci-llvm`. - #[cfg(not(feature = "bootstrap-self-test"))] - if !build.config.dry_run() && !build.build.is_msvc() && build.config.llvm_from_ci { - let builder = Builder::new(build); - let libcxx_version = builder.ensure(tool::LibcxxVersionTool { target: build.build }); - - match libcxx_version { - tool::LibcxxVersion::Gnu(version) => { - if LIBSTDCXX_MIN_VERSION_THRESHOLD > version { - eprintln!( - "\nYour system's libstdc++ version is too old for the `llvm.download-ci-llvm` option." - ); - eprintln!("Current version detected: '{}'", version); - eprintln!("Minimum required version: '{}'", LIBSTDCXX_MIN_VERSION_THRESHOLD); - eprintln!( - "Consider upgrading libstdc++ or disabling the `llvm.download-ci-llvm` option." - ); - eprintln!( - "If you choose to upgrade libstdc++, run `x clean` or delete `build/host/libcxx-version` manually after the upgrade." - ); - } - } - tool::LibcxxVersion::Llvm(_) => { - // FIXME: Handle libc++ version check. - } - } - } - - // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = !build.config.llvm_from_ci - && build - .hosts - .iter() - .map(|host| { - build.config.llvm_enabled(*host) - && build - .config - .target_config - .get(host) - .map(|config| config.llvm_config.is_none()) - .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); - - let need_cmake = building_llvm || build.config.any_sanitizers_to_build(); - if need_cmake && cmd_finder.maybe_have("cmake").is_none() { - eprintln!( - " -Couldn't find required command: cmake - -You should install cmake, or set `download-ci-llvm = true` in the -`[llvm]` section of `config.toml` to download LLVM rather -than building it. -" - ); - crate::exit!(1); - } - - build.config.python = build - .config - .python - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py - .or_else(|| cmd_finder.maybe_have("python")) - .or_else(|| cmd_finder.maybe_have("python3")) - .or_else(|| cmd_finder.maybe_have("python2")); - - build.config.nodejs = build - .config - .nodejs - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("node")) - .or_else(|| cmd_finder.maybe_have("nodejs")); - - build.config.npm = build - .config - .npm - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("npm")); - - build.config.gdb = build - .config - .gdb - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("gdb")); - - build.config.reuse = build - .config - .reuse - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("reuse")); - - let stage0_supported_target_list: HashSet = crate::utils::helpers::output( - command(&build.config.initial_rustc).args(["--print", "target-list"]).as_command_mut(), - ) - .lines() - .map(|s| s.to_string()) - .collect(); - - // We're gonna build some custom C code here and there, host triples - // also build some C++ shims for LLVM so we need a C++ compiler. - for target in &build.targets { - // On emscripten we don't actually need the C compiler to just - // build the target artifacts, only for testing. For the sake - // of easier bot configuration, just skip detection. - if target.contains("emscripten") { - continue; - } - - // We don't use a C compiler on wasm32 - if target.contains("wasm32") { - continue; - } - - // skip check for cross-targets - if skip_target_sanity && target != &build.build { - continue; - } - - // Ignore fake targets that are only used for unit tests in bootstrap. - if cfg!(not(feature = "bootstrap-self-test")) && !skip_target_sanity && !build.local_rebuild - { - let mut has_target = false; - let target_str = target.to_string(); - - let missing_targets_hashset: HashSet<_> = - STAGE0_MISSING_TARGETS.iter().map(|t| t.to_string()).collect(); - let duplicated_targets: Vec<_> = - stage0_supported_target_list.intersection(&missing_targets_hashset).collect(); - - if !duplicated_targets.is_empty() { - println!( - "Following targets supported from the stage0 compiler, please remove them from STAGE0_MISSING_TARGETS list." - ); - for duplicated_target in duplicated_targets { - println!(" {duplicated_target}"); - } - std::process::exit(1); - } - - // Check if it's a built-in target. - has_target |= stage0_supported_target_list.contains(&target_str); - has_target |= STAGE0_MISSING_TARGETS.contains(&target_str.as_str()); - - if !has_target { - // This might also be a custom target, so check the target file that could have been specified by the user. - if target.filepath().is_some_and(|p| p.exists()) { - has_target = true; - } else if let Some(custom_target_path) = env::var_os("RUST_TARGET_PATH") { - let mut target_filename = OsString::from(&target_str); - // Target filename ends with `.json`. - target_filename.push(".json"); - - // Recursively traverse through nested directories. - let walker = walkdir::WalkDir::new(custom_target_path).into_iter(); - for entry in walker.filter_map(|e| e.ok()) { - has_target |= entry.file_name() == target_filename; - } - } - } - - if !has_target { - panic!( - "No such target exists in the target list,\n\ - make sure to correctly specify the location \ - of the JSON specification file \ - for custom targets!\n\ - Use BOOTSTRAP_SKIP_TARGET_SANITY=1 to \ - bypass this check." - ); - } - } - - // sbf target relies on in-tree built llvm, - // which doesn't exist when this check runs - if !build.config.dry_run() && !target.contains("sbf") && !target.contains("bpf") { - cmd_finder.must_have(build.cc(*target)); - if let Some(ar) = build.ar(*target) { - cmd_finder.must_have(ar); - } - } - } - - if !build.config.dry_run() { - for host in &build.hosts { - cmd_finder.must_have(build.cxx(*host).unwrap()); - - if build.config.llvm_enabled(*host) { - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) - && !filecheck.exists() - && build.config.codegen_tests - { - panic!("FileCheck executable {filecheck:?} does not exist"); - } - } - } - } - - for target in &build.targets { - build - .config - .target_config - .entry(*target) - .or_insert_with(|| Target::from_triple(&target.triple)); - - if (target.contains("-none-") || target.contains("nvptx")) - && build.no_std(*target) == Some(false) - { - panic!("All the *-none-* and nvptx* targets are no-std targets") - } - - // skip check for cross-targets - if skip_target_sanity && target != &build.build { - continue; - } - - // Make sure musl-root is valid. - if target.contains("musl") && !target.contains("unikraft") { - // If this is a native target (host is also musl) and no musl-root is given, - // fall back to the system toolchain in /usr before giving up - if build.musl_root(*target).is_none() && build.config.build == *target { - let target = build.config.target_config.entry(*target).or_default(); - target.musl_root = Some("/usr".into()); - } - match build.musl_libdir(*target) { - Some(libdir) => { - if fs::metadata(libdir.join("libc.a")).is_err() { - panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); - } - } - None => panic!( - "when targeting MUSL either the rust.musl-root \ - option or the target.$TARGET.musl-root option must \ - be specified in config.toml" - ), - } - } - - if need_cmake && target.is_msvc() { - // There are three builds of cmake on windows: MSVC, MinGW, and - // Cygwin. The Cygwin build does not have generators for Visual - // Studio, so detect that here and error. - let out = - command("cmake").arg("--help").run_always().run_capture_stdout(build).stdout(); - if !out.contains("Visual Studio") { - panic!( - " -cmake does not support Visual Studio generators. - -This is likely due to it being an msys/cygwin build of cmake, -rather than the required windows version, built using MinGW -or Visual Studio. - -If you are building under msys2 try installing the mingw-w64-x86_64-cmake -package instead of cmake: - -$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake -" - ); - } - } - } - - if let Some(ref s) = build.config.ccache { - cmd_finder.must_have(s); - } -} diff --git a/standalonex/src/src/lib.rs b/standalonex/src/src/lib.rs deleted file mode 100644 index fe0d4f26..00000000 --- a/standalonex/src/src/lib.rs +++ /dev/null @@ -1,2033 +0,0 @@ -//! Implementation of bootstrap, the Rust build system. -//! -//! This module, and its descendants, are the implementation of the Rust build -//! system. Most of this build system is backed by Cargo but the outer layer -//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. The goals of bootstrap are: -//! -//! * To be an easily understandable, easily extensible, and maintainable build -//! system. -//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka -//! crates.io and Cargo. -//! * A standard interface to build across all platforms, including MSVC -//! -//! ## Further information -//! -//! More documentation can be found in each respective module below, and you can -//! also check out the `src/bootstrap/README.md` file for more information. - -use std::cell::{Cell, RefCell}; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fmt::Display; -use std::fs::{self, File}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::OnceLock; -use std::time::SystemTime; -use std::{env, io, str}; - -use build_helper::ci::gha; -use build_helper::exit; -use sha2::digest::Digest; -use termcolor::{ColorChoice, StandardStream, WriteColor}; -use utils::channel::GitInfo; -use utils::helpers::hex_encode; - -use crate::core::builder; -use crate::core::builder::{Builder, Kind}; -use crate::core::config::{DryRun, LldMode, LlvmLibunwind, Target, TargetSelection, flags}; -use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; -use crate::utils::helpers::{ - self, dir_is_empty, exe, libdir, mtime, output, set_file_times, symlink_dir, -}; - -mod core; -mod utils; - -pub use core::builder::PathSet; -pub use core::config::Config; -pub use core::config::flags::{Flags, Subcommand}; - -pub use utils::change_tracker::{ - CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes, -}; - -const LLVM_TOOLS: &[&str] = &[ - "llvm-cov", // used to generate coverage report - "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility - "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume - "llvm-objdump", // used to disassemble programs - "llvm-profdata", // used to inspect and merge files generated by profiles - "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide - "llvm-size", // used to prints the size of the linker sections of a program - "llvm-strip", // used to discard symbols from binary files to reduce their size - "llvm-ar", // used for creating and modifying archive files - "llvm-as", // used to convert LLVM assembly to LLVM bitcode - "llvm-dis", // used to disassemble LLVM bitcode - "llvm-link", // Used to link LLVM bitcode - "llc", // used to compile LLVM bytecode - "opt", // used to optimize LLVM bytecode -]; - -/// LLD file names for all flavors. -const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; - -/// Extra `--check-cfg` to add when building the compiler or tools -/// (Mode restriction, config name, config values (if any)) -#[allow(clippy::type_complexity)] // It's fine for hard-coded list and type is explained above. -const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ - (None, "bootstrap", None), - (Some(Mode::Rustc), "llvm_enzyme", None), - (Some(Mode::Codegen), "llvm_enzyme", None), - (Some(Mode::ToolRustc), "llvm_enzyme", None), - (Some(Mode::ToolRustc), "rust_analyzer", None), - (Some(Mode::ToolStd), "rust_analyzer", None), - // Any library specific cfgs like `target_os`, `target_arch` should be put in - // priority the `[lints.rust.unexpected_cfgs.check-cfg]` table - // in the appropriate `library/{std,alloc,core}/Cargo.toml` -]; - -/// A structure representing a Rust compiler. -/// -/// Each compiler has a `stage` that it is associated with and a `host` that -/// corresponds to the platform the compiler runs on. This structure is used as -/// a parameter to many methods below. -#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] -pub struct Compiler { - stage: u32, - host: TargetSelection, -} - -#[derive(PartialEq, Eq, Copy, Clone, Debug)] -pub enum DocTests { - /// Run normal tests and doc tests (default). - Yes, - /// Do not run any doc tests. - No, - /// Only run doc tests. - Only, -} - -pub enum GitRepo { - Rustc, - Llvm, -} - -/// Global configuration for the build system. -/// -/// This structure transitively contains all configuration for the build system. -/// All filesystem-encoded configuration is in `config`, all flags are in -/// `flags`, and then parsed or probed information is listed in the keys below. -/// -/// This structure is a parameter of almost all methods in the build system, -/// although most functions are implemented as free functions rather than -/// methods specifically on this structure itself (to make it easier to -/// organize). -#[derive(Clone)] -pub struct Build { - /// User-specified configuration from `config.toml`. - config: Config, - - // Version information - version: String, - - // Properties derived from the above configuration - src: PathBuf, - out: PathBuf, - bootstrap_out: PathBuf, - cargo_info: GitInfo, - rust_analyzer_info: GitInfo, - clippy_info: GitInfo, - miri_info: GitInfo, - rustfmt_info: GitInfo, - enzyme_info: GitInfo, - in_tree_llvm_info: GitInfo, - in_tree_gcc_info: GitInfo, - local_rebuild: bool, - fail_fast: bool, - doc_tests: DocTests, - verbosity: usize, - - /// Build triple for the pre-compiled snapshot compiler. - build: TargetSelection, - /// Which triples to produce a compiler toolchain for. - hosts: Vec, - /// Which triples to build libraries (core/alloc/std/test/proc_macro) for. - targets: Vec, - - initial_rustc: PathBuf, - initial_cargo: PathBuf, - initial_lld: PathBuf, - initial_libdir: PathBuf, - initial_sysroot: PathBuf, - - // Runtime state filled in later on - // C/C++ compilers and archiver for all targets - cc: RefCell>, - cxx: RefCell>, - ar: RefCell>, - ranlib: RefCell>, - // Miscellaneous - // allow bidirectional lookups: both name -> path and path -> name - crates: HashMap, - crate_paths: HashMap, - is_sudo: bool, - delayed_failures: RefCell>, - prerelease_version: Cell>, - - #[cfg(feature = "build-metrics")] - metrics: crate::utils::metrics::BuildMetrics, -} - -#[derive(Debug, Clone)] -struct Crate { - name: String, - deps: HashSet, - path: PathBuf, - has_lib: bool, - features: Vec, -} - -impl Crate { - fn local_path(&self, build: &Build) -> PathBuf { - self.path.strip_prefix(&build.config.src).unwrap().into() - } -} - -/// When building Rust various objects are handled differently. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum DependencyType { - /// Libraries originating from proc-macros. - Host, - /// Typical Rust libraries. - Target, - /// Non Rust libraries and objects shipped to ease usage of certain targets. - TargetSelfContained, -} - -/// The various "modes" of invoking Cargo. -/// -/// These entries currently correspond to the various output directories of the -/// build system, with each mod generating output in a different directory. -#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Mode { - /// Build the standard library, placing output in the "stageN-std" directory. - Std, - - /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. - Rustc, - - /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. - Codegen, - - /// Build a tool, placing output in the "stage0-bootstrap-tools" - /// directory. This is for miscellaneous sets of tools that are built - /// using the bootstrap stage0 compiler in its entirety (target libraries - /// and all). Typically these tools compile with stable Rust. - /// - /// Only works for stage 0. - ToolBootstrap, - - /// Build a tool which uses the locally built std, placing output in the - /// "stageN-tools" directory. Its usage is quite rare, mainly used by - /// compiletest which needs libtest. - ToolStd, - - /// Build a tool which uses the locally built rustc and the target std, - /// placing the output in the "stageN-tools" directory. This is used for - /// anything that needs a fully functional rustc, such as rustdoc, clippy, - /// cargo, rls, rustfmt, miri, etc. - ToolRustc, -} - -impl Mode { - pub fn is_tool(&self) -> bool { - matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) - } - - pub fn must_support_dlopen(&self) -> bool { - matches!(self, Mode::Std | Mode::Codegen) - } -} - -pub enum CLang { - C, - Cxx, -} - -macro_rules! forward { - ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { - impl Build { - $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { - self.config.$fn( $($param),* ) - } )+ - } - } -} - -forward! { - verbose(f: impl Fn()), - is_verbose() -> bool, - create(path: &Path, s: &str), - remove(f: &Path), - tempdir() -> PathBuf, - llvm_link_shared() -> bool, - download_rustc() -> bool, - initial_rustfmt() -> Option, -} - -impl Build { - /// Creates a new set of build configuration from the `flags` on the command - /// line and the filesystem `config`. - /// - /// By default all build output will be placed in the current directory. - pub fn new(mut config: Config) -> Build { - let src = config.src.clone(); - let out = config.out.clone(); - - #[cfg(unix)] - // keep this consistent with the equivalent check in x.py: - // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 - let is_sudo = match env::var_os("SUDO_USER") { - Some(_sudo_user) => { - // SAFETY: getuid() system call is always successful and no return value is reserved - // to indicate an error. - // - // For more context, see https://man7.org/linux/man-pages/man2/geteuid.2.html - let uid = unsafe { libc::getuid() }; - uid == 0 - } - None => false, - }; - #[cfg(not(unix))] - let is_sudo = false; - - let rust_info = config.rust_info.clone(); - let cargo_info = config.cargo_info.clone(); - let rust_analyzer_info = config.rust_analyzer_info.clone(); - let clippy_info = config.clippy_info.clone(); - let miri_info = config.miri_info.clone(); - let rustfmt_info = config.rustfmt_info.clone(); - let enzyme_info = config.enzyme_info.clone(); - let in_tree_llvm_info = config.in_tree_llvm_info.clone(); - let in_tree_gcc_info = config.in_tree_gcc_info.clone(); - - let initial_target_libdir_str = if config.dry_run() { - "/dummy/lib/path/to/lib/".to_string() - } else { - output( - Command::new(&config.initial_rustc) - .arg("--target") - .arg(config.build.rustc_target_arg()) - .arg("--print") - .arg("target-libdir"), - ) - }; - let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); - let initial_lld = initial_target_dir.join("bin").join("rust-lld"); - - let initial_sysroot = if config.dry_run() { - "/dummy".to_string() - } else { - output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) - } - .trim() - .to_string(); - - // FIXME(Zalathar): Determining this path occasionally fails locally for - // unknown reasons, so we print some extra context to help track down why. - let find_initial_libdir = || { - let initial_libdir = - initial_target_dir.parent()?.parent()?.strip_prefix(&initial_sysroot).ok()?; - Some(initial_libdir.to_path_buf()) - }; - let Some(initial_libdir) = find_initial_libdir() else { - panic!( - "couldn't determine `initial_libdir`: -- config.initial_rustc: {rustc:?} -- initial_target_libdir_str: {initial_target_libdir_str:?} -- initial_target_dir: {initial_target_dir:?} -- initial_sysroot: {initial_sysroot:?} -", - rustc = config.initial_rustc, - ); - }; - - let version = std::fs::read_to_string(src.join("src").join("version")) - .expect("failed to read src/version"); - let version = version.trim(); - - let mut bootstrap_out = std::env::current_exe() - .expect("could not determine path to running process") - .parent() - .unwrap() - .to_path_buf(); - // Since bootstrap is hardlink to deps/bootstrap-*, Solaris can sometimes give - // path with deps/ which is bad and needs to be avoided. - if bootstrap_out.ends_with("deps") { - bootstrap_out.pop(); - } - if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { - // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!( - "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", - bootstrap_out.display() - ) - } - - if rust_info.is_from_tarball() && config.description.is_none() { - config.description = Some("built from a source tarball".to_owned()); - } - - let mut build = Build { - initial_rustc: config.initial_rustc.clone(), - initial_cargo: config.initial_cargo.clone(), - initial_lld, - initial_libdir, - initial_sysroot: initial_sysroot.into(), - local_rebuild: config.local_rebuild, - fail_fast: config.cmd.fail_fast(), - doc_tests: config.cmd.doc_tests(), - verbosity: config.verbose, - - build: config.build, - hosts: config.hosts.clone(), - targets: config.targets.clone(), - - config, - version: version.to_string(), - src, - out, - bootstrap_out, - - cargo_info, - rust_analyzer_info, - clippy_info, - miri_info, - rustfmt_info, - enzyme_info, - in_tree_llvm_info, - in_tree_gcc_info, - cc: RefCell::new(HashMap::new()), - cxx: RefCell::new(HashMap::new()), - ar: RefCell::new(HashMap::new()), - ranlib: RefCell::new(HashMap::new()), - crates: HashMap::new(), - crate_paths: HashMap::new(), - is_sudo, - delayed_failures: RefCell::new(Vec::new()), - prerelease_version: Cell::new(None), - - #[cfg(feature = "build-metrics")] - metrics: crate::utils::metrics::BuildMetrics::init(), - }; - - // If local-rust is the same major.minor as the current version, then force a - // local-rebuild - let local_version_verbose = - output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); - let local_release = local_version_verbose - .lines() - .filter_map(|x| x.strip_prefix("release:")) - .next() - .unwrap() - .trim(); - if local_release.split('.').take(2).eq(version.split('.').take(2)) { - build.verbose(|| println!("auto-detected local-rebuild {local_release}")); - build.local_rebuild = true; - } - - build.verbose(|| println!("finding compilers")); - utils::cc_detect::find(&build); - // When running `setup`, the profile is about to change, so any requirements we have now may - // be different on the next invocation. Don't check for them until the next time x.py is - // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. - // - // Similarly, for `setup` we don't actually need submodules or cargo metadata. - if !matches!(build.config.cmd, Subcommand::Setup { .. }) { - build.verbose(|| println!("running sanity check")); - crate::core::sanity::check(&mut build); - - // Make sure we update these before gathering metadata so we don't get an error about missing - // Cargo.toml files. - let rust_submodules = ["library/backtrace", "library/stdarch"]; - for s in rust_submodules { - build.require_submodule( - s, - Some( - "The submodule is required for the standard library \ - and the main Cargo workspace.", - ), - ); - } - // Now, update all existing submodules. - build.update_existing_submodules(); - - build.verbose(|| println!("learning about cargo")); - crate::core::metadata::build(&mut build); - } - - // Create symbolic link to use host sysroot from a consistent path (e.g., in the rust-analyzer config file). - let build_triple = build.out.join(build.build); - t!(fs::create_dir_all(&build_triple)); - let host = build.out.join("host"); - if host.is_symlink() { - // Left over from a previous build; overwrite it. - // This matters if `build.build` has changed between invocations. - #[cfg(windows)] - t!(fs::remove_dir(&host)); - #[cfg(not(windows))] - t!(fs::remove_file(&host)); - } - t!( - symlink_dir(&build.config, &build_triple, &host), - format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) - ); - - build - } - - /// Updates a submodule, and exits with a failure if submodule management - /// is disabled and the submodule does not exist. - /// - /// The given submodule name should be its path relative to the root of - /// the main repository. - /// - /// The given `err_hint` will be shown to the user if the submodule is not - /// checked out and submodule management is disabled. - pub fn require_submodule(&self, submodule: &str, err_hint: Option<&str>) { - // When testing bootstrap itself, it is much faster to ignore - // submodules. Almost all Steps work fine without their submodules. - if cfg!(test) && !self.config.submodules() { - return; - } - self.config.update_submodule(submodule); - let absolute_path = self.config.src.join(submodule); - if dir_is_empty(&absolute_path) { - let maybe_enable = if !self.config.submodules() - && self.config.rust_info.is_managed_git_subrepository() - { - "\nConsider setting `build.submodules = true` or manually initializing the submodules." - } else { - "" - }; - let err_hint = err_hint.map_or_else(String::new, |e| format!("\n{e}")); - eprintln!( - "submodule {submodule} does not appear to be checked out, \ - but it is required for this step{maybe_enable}{err_hint}" - ); - exit!(1); - } - } - - /// Updates all submodules, and exits with an error if submodule - /// management is disabled and the submodule does not exist. - pub fn require_and_update_all_submodules(&self) { - for submodule in build_helper::util::parse_gitmodules(&self.src) { - self.require_submodule(submodule, None); - } - } - - /// If any submodule has been initialized already, sync it unconditionally. - /// This avoids contributors checking in a submodule change by accident. - fn update_existing_submodules(&self) { - // Avoid running git when there isn't a git checkout, or the user has - // explicitly disabled submodules in `config.toml`. - if !self.config.submodules() { - return; - } - let output = helpers::git(Some(&self.src)) - .args(["config", "--file"]) - .arg(".gitmodules") - .args(["--get-regexp", "path"]) - .run_capture(self) - .stdout(); - std::thread::scope(|s| { - // Look for `submodule.$name.path = $path` - // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` - for line in output.lines() { - let submodule = line.split_once(' ').unwrap().1; - let config = self.config.clone(); - s.spawn(move || { - Self::update_existing_submodule(&config, submodule); - }); - } - }); - } - - /// Updates the given submodule only if it's initialized already; nothing happens otherwise. - pub fn update_existing_submodule(config: &Config, submodule: &str) { - // Avoid running git when there isn't a git checkout. - if !config.submodules() { - return; - } - - if GitInfo::new(false, Path::new(submodule)).is_managed_git_subrepository() { - config.update_submodule(submodule); - } - } - - /// Executes the entire build, as configured by the flags and configuration. - pub fn build(&mut self) { - unsafe { - crate::utils::job::setup(self); - } - - // Download rustfmt early so that it can be used in rust-analyzer configs. - let _ = &builder::Builder::new(self).initial_rustfmt(); - - // hardcoded subcommands - match &self.config.cmd { - Subcommand::Format { check, all } => { - return core::build_steps::format::format( - &builder::Builder::new(self), - *check, - *all, - &self.config.paths, - ); - } - Subcommand::Suggest { run } => { - return core::build_steps::suggest::suggest(&builder::Builder::new(self), *run); - } - Subcommand::Perf { .. } => { - return core::build_steps::perf::perf(&builder::Builder::new(self)); - } - _ => (), - } - - if !self.config.dry_run() { - { - // We first do a dry-run. This is a sanity-check to ensure that - // steps don't do anything expensive in the dry-run. - self.config.dry_run = DryRun::SelfCheck; - let builder = builder::Builder::new(self); - builder.execute_cli(); - } - self.config.dry_run = DryRun::Disabled; - let builder = builder::Builder::new(self); - builder.execute_cli(); - } else { - let builder = builder::Builder::new(self); - builder.execute_cli(); - } - - // Check for postponed failures from `test --no-fail-fast`. - let failures = self.delayed_failures.borrow(); - if failures.len() > 0 { - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in failures.iter() { - eprintln!(" - {failure}\n"); - } - exit!(1); - } - - #[cfg(feature = "build-metrics")] - self.metrics.persist(self); - } - - /// Clear out `dir` if `input` is newer. - /// - /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { - let stamp = dir.join(".stamp"); - let mut cleared = false; - if mtime(&stamp) < mtime(input) { - self.verbose(|| println!("Dirty - {}", dir.display())); - let _ = fs::remove_dir_all(dir); - cleared = true; - } else if stamp.exists() { - return cleared; - } - t!(fs::create_dir_all(dir)); - t!(File::create(stamp)); - cleared - } - - fn rust_info(&self) -> &GitInfo { - &self.config.rust_info - } - - /// Gets the space-separated set of activated features for the standard library. - /// This can be configured with the `std-features` key in config.toml. - fn std_features(&self, target: TargetSelection) -> String { - let mut features: BTreeSet<&str> = - self.config.rust_std_features.iter().map(|s| s.as_str()).collect(); - - match self.config.llvm_libunwind(target) { - LlvmLibunwind::InTree => features.insert("llvm-libunwind"), - LlvmLibunwind::System => features.insert("system-llvm-libunwind"), - LlvmLibunwind::No => false, - }; - - if self.config.backtrace { - features.insert("backtrace"); - } - if self.config.profiler_enabled(target) { - features.insert("profiler"); - } - // Generate memcpy, etc. FIXME: Remove this once compiler-builtins - // automatically detects this target. - if target.contains("zkvm") { - features.insert("compiler-builtins-mem"); - } - - features.into_iter().collect::>().join(" ") - } - - /// Gets the space-separated set of activated features for the compiler. - fn rustc_features(&self, kind: Kind, target: TargetSelection, crates: &[String]) -> String { - let possible_features_by_crates: HashSet<_> = crates - .iter() - .flat_map(|krate| &self.crates[krate].features) - .map(std::ops::Deref::deref) - .collect(); - let check = |feature: &str| -> bool { - crates.is_empty() || possible_features_by_crates.contains(feature) - }; - let mut features = vec![]; - if self.config.jemalloc && check("jemalloc") { - features.push("jemalloc"); - } - if (self.config.llvm_enabled(target) || kind == Kind::Check) && check("llvm") { - features.push("llvm"); - } - // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` - if self.config.rust_randomize_layout { - features.push("rustc_randomized_layouts"); - } - - // If debug logging is on, then we want the default for tracing: - // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 - // which is everything (including debug/trace/etc.) - // if its unset, if debug_assertions is on, then debug_logging will also be on - // as well as tracing *ignoring* this feature when debug_assertions is on - if !self.config.rust_debug_logging && check("max_level_info") { - features.push("max_level_info"); - } - - features.join(" ") - } - - /// Component directory that Cargo will produce output into (e.g. - /// release/debug) - fn cargo_dir(&self) -> &'static str { - if self.config.rust_optimize.is_release() { "release" } else { "debug" } - } - - fn tools_dir(&self, compiler: Compiler) -> PathBuf { - let out = self.out.join(compiler.host).join(format!("stage{}-tools-bin", compiler.stage)); - t!(fs::create_dir_all(&out)); - out - } - - /// Returns the root directory for all output generated in a particular - /// stage when running with a particular host compiler. - /// - /// The mode indicates what the root directory is for. - fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { - let suffix = match mode { - Mode::Std => "-std", - Mode::Rustc => "-rustc", - Mode::Codegen => "-codegen", - Mode::ToolBootstrap => "-bootstrap-tools", - Mode::ToolStd | Mode::ToolRustc => "-tools", - }; - self.out.join(compiler.host).join(format!("stage{}{}", compiler.stage, suffix)) - } - - /// Returns the root output directory for all Cargo output in a given stage, - /// running a particular compiler, whether or not we're building the - /// standard library, and targeting the specified architecture. - fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { - self.stage_out(compiler, mode).join(target).join(self.cargo_dir()) - } - - /// Root output directory of LLVM for `target` - /// - /// Note that if LLVM is configured externally then the directory returned - /// will likely be empty. - fn llvm_out(&self, target: TargetSelection) -> PathBuf { - if self.config.llvm_from_ci && self.config.build == target { - self.config.ci_llvm_root() - } else { - self.out.join(target).join("llvm") - } - } - - fn enzyme_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("enzyme") - } - - fn gcc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("gcc") - } - - fn lld_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("lld") - } - - /// Output directory for all documentation for a target - fn doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("doc") - } - - /// Output directory for all JSON-formatted documentation for a target - fn json_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("json-doc") - } - - fn test_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("test") - } - - /// Output directory for all documentation for a target - fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("compiler-doc") - } - - /// Output directory for some generated md crate documentation for a target (temporary) - fn md_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("md-doc") - } - - /// Returns `true` if this is an external version of LLVM not managed by bootstrap. - /// In particular, we expect llvm sources to be available when this is false. - /// - /// NOTE: this is not the same as `!is_rust_llvm` when `llvm_has_patches` is set. - fn is_system_llvm(&self, target: TargetSelection) -> bool { - match self.config.target_config.get(&target) { - Some(Target { llvm_config: Some(_), .. }) => { - let ci_llvm = self.config.llvm_from_ci && target == self.config.build; - !ci_llvm - } - // We're building from the in-tree src/llvm-project sources. - Some(Target { llvm_config: None, .. }) => false, - None => false, - } - } - - /// Returns `true` if this is our custom, patched, version of LLVM. - /// - /// This does not necessarily imply that we're managing the `llvm-project` submodule. - fn is_rust_llvm(&self, target: TargetSelection) -> bool { - match self.config.target_config.get(&target) { - // We're using a user-controlled version of LLVM. The user has explicitly told us whether the version has our patches. - // (They might be wrong, but that's not a supported use-case.) - // In particular, this tries to support `submodules = false` and `patches = false`, for using a newer version of LLVM that's not through `rust-lang/llvm-project`. - Some(Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, - // The user hasn't promised the patches match. - // This only has our patches if it's downloaded from CI or built from source. - _ => !self.is_system_llvm(target), - } - } - - /// Returns the path to llvm/bin - fn llvm_bin(&self, target: TargetSelection) -> PathBuf { - let target_config = self.config.target_config.get(&target); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - let llvm_bindir = output(Command::new(s).arg("--bindir")); - PathBuf::from(llvm_bindir.trim()) - } else { - self.llvm_out(self.config.build).join("bin") - } - } - - /// Returns the path to `FileCheck` binary for the specified target - fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { - let target_config = self.config.target_config.get(&target); - if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { - s.to_path_buf() - } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - let llvm_bindir = command(s).arg("--bindir").run_capture_stdout(self).stdout(); - let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); - if filecheck.exists() { - filecheck - } else { - // On Fedora the system LLVM installs FileCheck in the - // llvm subdirectory of the libdir. - let llvm_libdir = command(s).arg("--libdir").run_capture_stdout(self).stdout(); - let lib_filecheck = - Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); - if lib_filecheck.exists() { - lib_filecheck - } else { - // Return the most normal file name, even though - // it doesn't exist, so that any error message - // refers to that. - filecheck - } - } - } else { - let base = self.llvm_out(target).join("build"); - let base = if !self.ninja() && target.is_msvc() { - if self.config.llvm_optimize { - if self.config.llvm_release_debuginfo { - base.join("RelWithDebInfo") - } else { - base.join("Release") - } - } else { - base.join("Debug") - } - } else { - base - }; - base.join("bin").join(exe("FileCheck", target)) - } - } - - /// Directory for libraries built from C/C++ code and shared between stages. - fn native_dir(&self, target: TargetSelection) -> PathBuf { - self.out.join(target).join("native") - } - - /// Root output directory for rust_test_helpers library compiled for - /// `target` - fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { - self.native_dir(target).join("rust-test-helpers") - } - - /// Adds the `RUST_TEST_THREADS` env var if necessary - fn add_rust_test_threads(&self, cmd: &mut BootstrapCommand) { - if env::var_os("RUST_TEST_THREADS").is_none() { - cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); - } - } - - /// Returns the libdir of the snapshot compiler. - fn rustc_snapshot_libdir(&self) -> PathBuf { - self.rustc_snapshot_sysroot().join(libdir(self.config.build)) - } - - /// Returns the sysroot of the snapshot compiler. - fn rustc_snapshot_sysroot(&self) -> &Path { - static SYSROOT_CACHE: OnceLock = OnceLock::new(); - SYSROOT_CACHE.get_or_init(|| { - let mut rustc = Command::new(&self.initial_rustc); - rustc.args(["--print", "sysroot"]); - output(&mut rustc).trim().into() - }) - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - fn run( - &self, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> CommandOutput { - command.mark_as_executed(); - if self.config.dry_run() && !command.run_always { - return CommandOutput::default(); - } - - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - - self.verbose(|| { - println!("running: {command:?} (created at {created_at}, executed at {executed_at})") - }); - - let cmd = command.as_command_mut(); - cmd.stdout(stdout.stdio()); - cmd.stderr(stderr.stdio()); - - let output = cmd.output(); - - use std::fmt::Write; - - let mut message = String::new(); - let output: CommandOutput = match output { - // Command has succeeded - Ok(output) if output.status.success() => { - CommandOutput::from_output(output, stdout, stderr) - } - // Command has started, but then it failed - Ok(output) => { - writeln!( - message, - r#" -Command {command:?} did not execute successfully. -Expected success, got {} -Created at: {created_at} -Executed at: {executed_at}"#, - output.status, - ) - .unwrap(); - - let output: CommandOutput = CommandOutput::from_output(output, stdout, stderr); - - // If the output mode is OutputMode::Capture, we can now print the output. - // If it is OutputMode::Print, then the output has already been printed to - // stdout/stderr, and we thus don't have anything captured to print anyway. - if stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); - } - if stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); - } - output - } - // The command did not even start - Err(e) => { - writeln!( - message, - "\n\nCommand {command:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}" - ) - .unwrap(); - CommandOutput::did_not_start(stdout, stderr) - } - }; - - let fail = |message: &str, output: CommandOutput| -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = - output.stdout_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = - output.stderr_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDERR:\n{stderr}\n"); - } - println!("Command {command:?} has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } - } - exit!(1); - }; - - if !output.is_success() { - match command.failure_behavior { - BehaviorOnFailure::DelayFail => { - if self.fail_fast { - fail(&message, output); - } - - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(message); - } - BehaviorOnFailure::Exit => { - fail(&message, output); - } - BehaviorOnFailure::Ignore => { - // If failures are allowed, either the error has been printed already - // (OutputMode::Print) or the user used a capture output mode and wants to - // handle the error output on their own. - } - } - } - output - } - - /// Check if verbosity is greater than the `level` - pub fn is_verbose_than(&self, level: usize) -> bool { - self.verbosity > level - } - - /// Runs a function if verbosity is greater than `level`. - fn verbose_than(&self, level: usize, f: impl Fn()) { - if self.is_verbose_than(level) { - f() - } - } - - fn info(&self, msg: &str) { - match self.config.dry_run { - DryRun::SelfCheck => (), - DryRun::Disabled | DryRun::UserSelected => { - println!("{msg}"); - } - } - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_clippy( - &self, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Clippy, self.config.stage, what, self.config.build, target) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_check( - &self, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Check, self.config.stage, what, self.config.build, target) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_doc( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into> + Copy, - ) -> Option { - self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_build( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Build, compiler.stage, what, compiler.host, target) - } - - /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. - /// - /// [`Step`]: crate::core::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: impl Into>, - target: impl Into>, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); - let msg = if let Some(target) = target.into() { - let host = host.into().unwrap(); - if host == target { - msg(format_args!(" ({target})")) - } else { - msg(format_args!(" ({host} -> {target})")) - } - } else { - msg(format_args!("")) - }; - self.group(&msg) - } - - /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. - /// - /// [`Step`]: crate::core::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_unstaged( - &self, - action: impl Into, - what: impl Display, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = format!("{action} {what} for {target}"); - self.group(&msg) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_sysroot_tool( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: TargetSelection, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} {what} {fmt}"); - let msg = if host == target { - msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) - } else { - msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) - }; - self.group(&msg) - } - - #[track_caller] - fn group(&self, msg: &str) -> Option { - match self.config.dry_run { - DryRun::SelfCheck => None, - DryRun::Disabled | DryRun::UserSelected => Some(gha::group(msg)), - } - } - - /// Returns the number of parallel jobs that have been configured for this - /// build. - fn jobs(&self) -> u32 { - self.config.jobs.unwrap_or_else(|| { - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }) - } - - fn debuginfo_map_to(&self, which: GitRepo) -> Option { - if !self.config.rust_remap_debuginfo { - return None; - } - - match which { - GitRepo::Rustc => { - let sha = self.rust_sha().unwrap_or(&self.version); - Some(format!("/rustc/{sha}")) - } - GitRepo::Llvm => Some(String::from("/rustc/llvm")), - } - } - - /// Returns the path to the C compiler for the target specified. - fn cc(&self, target: TargetSelection) -> PathBuf { - if self.config.dry_run() { - return PathBuf::new(); - } - self.cc.borrow()[&target].path().into() - } - - /// Returns a list of flags to pass to the C compiler for the target - /// specified. - fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { - if self.config.dry_run() { - return Vec::new(); - } - let base = match c { - CLang::C => self.cc.borrow()[&target].clone(), - CLang::Cxx => self.cxx.borrow()[&target].clone(), - }; - - // Filter out -O and /O (the optimization flags) that we picked up from - // cc-rs because the build scripts will determine that for themselves. - let mut base = base - .args() - .iter() - .map(|s| s.to_string_lossy().into_owned()) - .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) - .collect::>(); - - // If we're compiling C++ on macOS then we add a flag indicating that - // we want libc++ (more filled out than libstdc++), ensuring that - // LLVM/etc are all properly compiled. - if matches!(c, CLang::Cxx) && target.contains("apple-darwin") { - base.push("-stdlib=libc++".into()); - } - - // Work around an apparently bad MinGW / GCC optimization, - // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html - // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 - if &*target.triple == "i686-pc-windows-gnu" { - base.push("-fno-omit-frame-pointer".into()); - } - - if let Some(map_to) = self.debuginfo_map_to(which) { - let map = format!("{}={}", self.src.display(), map_to); - let cc = self.cc(target); - if cc.ends_with("clang") || cc.ends_with("gcc") { - base.push(format!("-fdebug-prefix-map={map}")); - } else if cc.ends_with("clang-cl.exe") { - base.push("-Xclang".into()); - base.push(format!("-fdebug-prefix-map={map}")); - } - } - base - } - - /// Returns the path to the `ar` archive utility for the target specified. - fn ar(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ar.borrow().get(&target).cloned() - } - - /// Returns the path to the `ranlib` utility for the target specified. - fn ranlib(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ranlib.borrow().get(&target).cloned() - } - - /// Returns the path to the C++ compiler for the target specified. - fn cxx(&self, target: TargetSelection) -> Result { - if self.config.dry_run() { - return Ok(PathBuf::new()); - } - match self.cxx.borrow().get(&target) { - Some(p) => Ok(p.path().into()), - None => Err(format!("target `{target}` is not configured as a host, only as a target")), - } - } - - /// Returns the path to the linker for the given target if it needs to be overridden. - fn linker(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return Some(PathBuf::new()); - } - if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) - { - Some(linker) - } else if target.contains("vxworks") { - // need to use CXX compiler as linker to resolve the exception functions - // that are only existed in CXX libraries - Some(self.cxx.borrow()[&target].path().into()) - } else if target != self.config.build - && helpers::use_host_linker(target) - && !target.is_msvc() - { - Some(self.cc(target)) - } else if self.config.lld_mode.is_used() - && self.is_lld_direct_linker(target) - && self.build == target - { - match self.config.lld_mode { - LldMode::SelfContained => Some(self.initial_lld.clone()), - LldMode::External => Some("lld".into()), - LldMode::Unused => None, - } - } else { - None - } - } - - // Is LLD configured directly through `-Clinker`? - // Only MSVC targets use LLD directly at the moment. - fn is_lld_direct_linker(&self, target: TargetSelection) -> bool { - target.is_msvc() - } - - /// Returns if this target should statically link the C runtime, if specified - fn crt_static(&self, target: TargetSelection) -> Option { - if target.contains("pc-windows-msvc") { - Some(true) - } else { - self.config.target_config.get(&target).and_then(|t| t.crt_static) - } - } - - /// Returns the "musl root" for this `target`, if defined - fn musl_root(&self, target: TargetSelection) -> Option<&Path> { - self.config - .target_config - .get(&target) - .and_then(|t| t.musl_root.as_ref()) - .or(self.config.musl_root.as_ref()) - .map(|p| &**p) - } - - /// Returns the "musl libdir" for this `target`. - fn musl_libdir(&self, target: TargetSelection) -> Option { - let t = self.config.target_config.get(&target)?; - if let libdir @ Some(_) = &t.musl_libdir { - return libdir.clone(); - } - self.musl_root(target).map(|root| root.join("lib")) - } - - /// Returns the `lib` directory for the WASI target specified, if - /// configured. - /// - /// This first consults `wasi-root` as configured in per-target - /// configuration, and failing that it assumes that `$WASI_SDK_PATH` is - /// set in the environment, and failing that `None` is returned. - fn wasi_libdir(&self, target: TargetSelection) -> Option { - let configured = - self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p); - if let Some(path) = configured { - return Some(path.join("lib").join(target.to_string())); - } - let mut env_root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?); - env_root.push("share"); - env_root.push("wasi-sysroot"); - env_root.push("lib"); - env_root.push(target.to_string()); - Some(env_root) - } - - /// Returns `true` if this is a no-std `target`, if defined - fn no_std(&self, target: TargetSelection) -> Option { - self.config.target_config.get(&target).map(|t| t.no_std) - } - - /// Returns `true` if the target will be tested using the `remote-test-client` - /// and `remote-test-server` binaries. - fn remote_tested(&self, target: TargetSelection) -> bool { - self.qemu_rootfs(target).is_some() - || target.contains("android") - || env::var_os("TEST_DEVICE_ADDR").is_some() - } - - /// Returns an optional "runner" to pass to `compiletest` when executing - /// test binaries. - /// - /// An example of this would be a WebAssembly runtime when testing the wasm - /// targets. - fn runner(&self, target: TargetSelection) -> Option { - let configured_runner = - self.config.target_config.get(&target).and_then(|t| t.runner.as_ref()).map(|p| &**p); - if let Some(runner) = configured_runner { - return Some(runner.to_owned()); - } - - if target.starts_with("wasm") && target.contains("wasi") { - self.default_wasi_runner(target) - } else { - None - } - } - - /// When a `runner` configuration is not provided and a WASI-looking target - /// is being tested this is consulted to prove the environment to see if - /// there's a runtime already lying around that seems reasonable to use. - fn default_wasi_runner(&self, target: TargetSelection) -> Option { - let mut finder = crate::core::sanity::Finder::new(); - - // Look for Wasmtime, and for its default options be sure to disable - // its caching system since we're executing quite a lot of tests and - // ideally shouldn't pollute the cache too much. - if let Some(path) = finder.maybe_have("wasmtime") { - if let Ok(mut path) = path.into_os_string().into_string() { - path.push_str(" run -C cache=n --dir ."); - // Make sure that tests have access to RUSTC_BOOTSTRAP. This (for example) is - // required for libtest to work on beta/stable channels. - // - // NB: with Wasmtime 20 this can change to `-S inherit-env` to - // inherit the entire environment rather than just this single - // environment variable. - path.push_str(" --env RUSTC_BOOTSTRAP"); - - if target.contains("wasip2") { - path.push_str(" --wasi inherit-network --wasi allow-ip-name-lookup"); - } - - return Some(path); - } - } - - None - } - - /// Returns whether the specified tool is configured as part of this build. - /// - /// This requires that both the `extended` key is set and the `tools` key is - /// either unset or specifically contains the specified tool. - fn tool_enabled(&self, tool: &str) -> bool { - if !self.config.extended { - return false; - } - match &self.config.tools { - Some(set) => set.contains(tool), - None => true, - } - } - - /// Returns the root of the "rootfs" image that this target will be using, - /// if one was configured. - /// - /// If `Some` is returned then that means that tests for this target are - /// emulated with QEMU and binaries will need to be shipped to the emulator. - fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) - } - - /// Path to the python interpreter to use - fn python(&self) -> &Path { - if self.config.build.ends_with("apple-darwin") { - // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the - // LLDB plugin's compiled module which only works with the system python - // (namely not Homebrew-installed python) - Path::new("/usr/bin/python3") - } else { - self.config - .python - .as_ref() - .expect("python is required for running LLDB or rustdoc tests") - } - } - - /// Temporary directory that extended error information is emitted to. - fn extended_error_dir(&self) -> PathBuf { - self.out.join("tmp/extended-error-metadata") - } - - /// Tests whether the `compiler` compiling for `target` should be forced to - /// use a stage1 compiler instead. - /// - /// Currently, by default, the build system does not perform a "full - /// bootstrap" by default where we compile the compiler three times. - /// Instead, we compile the compiler two times. The final stage (stage2) - /// just copies the libraries from the previous stage, which is what this - /// method detects. - /// - /// Here we return `true` if: - /// - /// * The build isn't performing a full bootstrap - /// * The `compiler` is in the final stage, 2 - /// * We're not cross-compiling, so the artifacts are already available in - /// stage1 - /// - /// When all of these conditions are met the build will lift artifacts from - /// the previous stage forward. - fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { - !self.config.full_bootstrap - && !self.config.download_rustc() - && stage >= 2 - && (self.hosts.iter().any(|h| *h == target) || target == self.build) - } - - /// Checks whether the `compiler` compiling for `target` should be forced to - /// use a stage2 compiler instead. - /// - /// When we download the pre-compiled version of rustc and compiler stage is >= 2, - /// it should be forced to use a stage2 compiler. - fn force_use_stage2(&self, stage: u32) -> bool { - self.config.download_rustc() && stage >= 2 - } - - /// Given `num` in the form "a.b.c" return a "release string" which - /// describes the release version number. - /// - /// For example on nightly this returns "a.b.c-nightly", on beta it returns - /// "a.b.c-beta.1" and on stable it just returns "a.b.c". - fn release(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => { - if !self.config.omit_git_hash { - format!("{}-beta.{}", num, self.beta_prerelease_version()) - } else { - format!("{num}-beta") - } - } - "nightly" => format!("{num}-nightly"), - _ => format!("{num}-dev"), - } - } - - fn beta_prerelease_version(&self) -> u32 { - fn extract_beta_rev_from_file>(version_file: P) -> Option { - let version = fs::read_to_string(version_file).ok()?; - - helpers::extract_beta_rev(&version) - } - - if let Some(s) = self.prerelease_version.get() { - return s; - } - - // First check if there is a version file available. - // If available, we read the beta revision from that file. - // This only happens when building from a source tarball when Git should not be used. - let count = extract_beta_rev_from_file(self.src.join("version")).unwrap_or_else(|| { - // Figure out how many merge commits happened since we branched off master. - // That's our beta number! - // (Note that we use a `..` range, not the `...` symmetric difference.) - helpers::git(Some(&self.src)) - .arg("rev-list") - .arg("--count") - .arg("--merges") - .arg(format!( - "refs/remotes/origin/{}..HEAD", - self.config.stage0_metadata.config.nightly_branch - )) - .run_always() - .run_capture(self) - .stdout() - }); - let n = count.trim().parse().unwrap(); - self.prerelease_version.set(Some(n)); - n - } - - /// Returns the value of `release` above for Rust itself. - fn rust_release(&self) -> String { - self.release(&self.version) - } - - /// Returns the "package version" for a component given the `num` release - /// number. - /// - /// The package version is typically what shows up in the names of tarballs. - /// For channels like beta/nightly it's just the channel name, otherwise - /// it's the `num` provided. - fn package_vers(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => "beta".to_string(), - "nightly" => "nightly".to_string(), - _ => format!("{num}-dev"), - } - } - - /// Returns the value of `package_vers` above for Rust itself. - fn rust_package_vers(&self) -> String { - self.package_vers(&self.version) - } - - /// Returns the `version` string associated with this compiler for Rust - /// itself. - /// - /// Note that this is a descriptive string which includes the commit date, - /// sha, version, etc. - fn rust_version(&self) -> String { - let mut version = self.rust_info().version(self, &self.version); - if let Some(ref s) = self.config.description { - if !s.is_empty() { - version.push_str(" ("); - version.push_str(s); - version.push(')'); - } - } - version - } - - /// Returns the full commit hash. - fn rust_sha(&self) -> Option<&str> { - self.rust_info().sha() - } - - /// Returns the `a.b.c` version that the given package is at. - fn release_num(&self, package: &str) -> String { - let toml_file_name = self.src.join(format!("src/tools/{package}/Cargo.toml")); - let toml = t!(fs::read_to_string(toml_file_name)); - for line in toml.lines() { - if let Some(stripped) = - line.strip_prefix("version = \"").and_then(|s| s.strip_suffix('"')) - { - return stripped.to_owned(); - } - } - - panic!("failed to find version in {package}'s Cargo.toml") - } - - /// Returns `true` if unstable features should be enabled for the compiler - /// we're building. - fn unstable_features(&self) -> bool { - !matches!(&self.config.channel[..], "stable" | "beta") - } - - /// Returns a Vec of all the dependencies of the given root crate, - /// including transitive dependencies and the root itself. Only includes - /// "local" crates (those in the local source tree, not from a registry). - fn in_tree_crates(&self, root: &str, target: Option) -> Vec<&Crate> { - let mut ret = Vec::new(); - let mut list = vec![root.to_owned()]; - let mut visited = HashSet::new(); - while let Some(krate) = list.pop() { - let krate = self - .crates - .get(&krate) - .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); - ret.push(krate); - for dep in &krate.deps { - if !self.crates.contains_key(dep) { - // Ignore non-workspace members. - continue; - } - // Don't include optional deps if their features are not - // enabled. Ideally this would be computed from `cargo - // metadata --features â€Ļ`, but that is somewhat slow. In - // the future, we may want to consider just filtering all - // build and dev dependencies in metadata::build. - if visited.insert(dep) - && (dep != "profiler_builtins" - || target - .map(|t| self.config.profiler_enabled(t)) - .unwrap_or_else(|| self.config.any_profiler_enabled())) - && (dep != "rustc_codegen_llvm" - || self.config.hosts.iter().any(|host| self.config.llvm_enabled(*host))) - { - list.push(dep.clone()); - } - } - } - ret.sort_unstable_by_key(|krate| krate.name.clone()); // reproducible order needed for tests - ret - } - - fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { - if self.config.dry_run() { - return Vec::new(); - } - - if !stamp.exists() { - eprintln!( - "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", - stamp.display() - ); - crate::exit!(1); - } - - let mut paths = Vec::new(); - let contents = t!(fs::read(stamp), &stamp); - // This is the method we use for extracting paths from the stamp file passed to us. See - // run_cargo for more information (in compile.rs). - for part in contents.split(|b| *b == 0) { - if part.is_empty() { - continue; - } - let dependency_type = match part[0] as char { - 'h' => DependencyType::Host, - 's' => DependencyType::TargetSelfContained, - 't' => DependencyType::Target, - _ => unreachable!(), - }; - let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); - paths.push((path, dependency_type)); - } - paths - } - - /// Copies a file from `src` to `dst`. - /// - /// If `src` is a symlink, `src` will be resolved to the actual path - /// and copied to `dst` instead of the symlink itself. - pub fn resolve_symlink_and_copy(&self, src: &Path, dst: &Path) { - self.copy_link_internal(src, dst, true); - } - - /// Links a file from `src` to `dst`. - /// Attempts to use hard links if possible, falling back to copying. - /// You can neither rely on this being a copy nor it being a link, - /// so do not write to dst. - pub fn copy_link(&self, src: &Path, dst: &Path) { - self.copy_link_internal(src, dst, false); - } - - fn copy_link_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { - if self.config.dry_run() { - return; - } - self.verbose_than(1, || println!("Copy/Link {src:?} to {dst:?}")); - if src == dst { - return; - } - if let Err(e) = fs::remove_file(dst) { - if cfg!(windows) && e.kind() != io::ErrorKind::NotFound { - // workaround for https://github.com/rust-lang/rust/issues/127126 - // if removing the file fails, attempt to rename it instead. - let now = t!(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)); - let _ = fs::rename(dst, format!("{}-{}", dst.display(), now.as_nanos())); - } - } - let metadata = t!(src.symlink_metadata(), format!("src = {}", src.display())); - let mut src = src.to_path_buf(); - if metadata.file_type().is_symlink() { - if dereference_symlinks { - src = t!(fs::canonicalize(src)); - } else { - let link = t!(fs::read_link(src)); - t!(self.symlink_file(link, dst)); - return; - } - } - if let Ok(()) = fs::hard_link(&src, dst) { - // Attempt to "easy copy" by creating a hard link (symlinks are priviledged on windows), - // but if that fails just fall back to a slow `copy` operation. - } else { - if let Err(e) = fs::copy(&src, dst) { - panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) - } - t!(fs::set_permissions(dst, metadata.permissions())); - - // Restore file times because changing permissions on e.g. Linux using `chmod` can cause - // file access time to change. - let file_times = fs::FileTimes::new() - .set_accessed(t!(metadata.accessed())) - .set_modified(t!(metadata.modified())); - t!(set_file_times(dst, file_times)); - } - } - - /// Links the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. - /// Will attempt to use hard links if possible and fall back to copying. - pub fn cp_link_r(&self, src: &Path, dst: &Path) { - if self.config.dry_run() { - return; - } - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - if t!(f.file_type()).is_dir() { - t!(fs::create_dir_all(&dst)); - self.cp_link_r(&path, &dst); - } else { - self.copy_link(&path, &dst); - } - } - } - - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. - /// Will attempt to use hard links if possible and fall back to copying. - /// Unwanted files or directories can be skipped - /// by returning `false` from the filter function. - pub fn cp_link_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { - // Immediately recurse with an empty relative path - self.cp_link_filtered_recurse(src, dst, Path::new(""), filter) - } - - // Inner function does the actual work - fn cp_link_filtered_recurse( - &self, - src: &Path, - dst: &Path, - relative: &Path, - filter: &dyn Fn(&Path) -> bool, - ) { - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - let relative = relative.join(name); - // Only copy file or directory if the filter function returns true - if filter(&relative) { - if t!(f.file_type()).is_dir() { - let _ = fs::remove_dir_all(&dst); - self.create_dir(&dst); - self.cp_link_filtered_recurse(&path, &dst, &relative, filter); - } else { - let _ = fs::remove_file(&dst); - self.copy_link(&path, &dst); - } - } - } - } - - fn copy_link_to_folder(&self, src: &Path, dest_folder: &Path) { - let file_name = src.file_name().unwrap(); - let dest = dest_folder.join(file_name); - self.copy_link(src, &dest); - } - - fn install(&self, src: &Path, dstdir: &Path, perms: u32) { - if self.config.dry_run() { - return; - } - let dst = dstdir.join(src.file_name().unwrap()); - self.verbose_than(1, || println!("Install {src:?} to {dst:?}")); - t!(fs::create_dir_all(dstdir)); - if !src.exists() { - panic!("ERROR: File \"{}\" not found!", src.display()); - } - self.copy_link_internal(src, &dst, true); - chmod(&dst, perms); - } - - fn read(&self, path: &Path) -> String { - if self.config.dry_run() { - return String::new(); - } - t!(fs::read_to_string(path)) - } - - fn create_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::create_dir_all(dir)) - } - - fn remove_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::remove_dir_all(dir)) - } - - fn read_dir(&self, dir: &Path) -> impl Iterator { - let iter = match fs::read_dir(dir) { - Ok(v) => v, - Err(_) if self.config.dry_run() => return vec![].into_iter(), - Err(err) => panic!("could not read dir {dir:?}: {err:?}"), - }; - iter.map(|e| t!(e)).collect::>().into_iter() - } - - fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { - #[cfg(unix)] - use std::os::unix::fs::symlink as symlink_file; - #[cfg(windows)] - use std::os::windows::fs::symlink_file; - if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } - } - - /// Returns if config.ninja is enabled, and checks for ninja existence, - /// exiting with a nicer error message if not. - fn ninja(&self) -> bool { - let mut cmd_finder = crate::core::sanity::Finder::new(); - - if self.config.ninja_in_file { - // Some Linux distros rename `ninja` to `ninja-build`. - // CMake can work with either binary name. - if cmd_finder.maybe_have("ninja-build").is_none() - && cmd_finder.maybe_have("ninja").is_none() - { - eprintln!( - " -Couldn't find required command: ninja (or ninja-build) - -You should install ninja as described at -, -or set `ninja = false` in the `[llvm]` section of `config.toml`. -Alternatively, set `download-ci-llvm = true` in that `[llvm]` section -to download LLVM rather than building it. -" - ); - exit!(1); - } - } - - // If ninja isn't enabled but we're building for MSVC then we try - // doubly hard to enable it. It was realized in #43767 that the msbuild - // CMake generator for MSVC doesn't respect configuration options like - // disabling LLVM assertions, which can often be quite important! - // - // In these cases we automatically enable Ninja if we find it in the - // environment. - if !self.config.ninja_in_file - && self.config.build.is_msvc() - && cmd_finder.maybe_have("ninja").is_some() - { - return true; - } - - self.config.ninja_in_file - } - - pub fn colored_stdout R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) - } - - pub fn colored_stderr R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) - } - - fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R - where - C: Fn(ColorChoice) -> StandardStream, - F: FnOnce(&mut dyn WriteColor) -> R, - { - let choice = match self.config.color { - flags::Color::Always => ColorChoice::Always, - flags::Color::Never => ColorChoice::Never, - flags::Color::Auto if !is_tty => ColorChoice::Never, - flags::Color::Auto => ColorChoice::Auto, - }; - let mut stream = constructor(choice); - let result = f(&mut stream); - stream.reset().unwrap(); - result - } -} - -#[cfg(unix)] -fn chmod(path: &Path, perms: u32) { - use std::os::unix::fs::*; - t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); -} -#[cfg(windows)] -fn chmod(_path: &Path, _perms: u32) {} - -impl Compiler { - pub fn with_stage(mut self, stage: u32) -> Compiler { - self.stage = stage; - self - } - - /// Returns `true` if this is a snapshot compiler for `build`'s configuration - pub fn is_snapshot(&self, build: &Build) -> bool { - self.stage == 0 && self.host == build.build - } -} - -fn envify(s: &str) -> String { - s.chars() - .map(|c| match c { - '-' => '_', - c => c, - }) - .flat_map(|c| c.to_uppercase()) - .collect() -} - -/// Computes a hash representing the state of a repository/submodule and additional input. -/// -/// It uses `git diff` for the actual changes, and `git status` for including the untracked -/// files in the specified directory. The additional input is also incorporated into the -/// computation of the hash. -/// -/// # Parameters -/// -/// - `dir`: A reference to the directory path of the target repository/submodule. -/// - `additional_input`: An additional input to be included in the hash. -/// -/// # Panics -/// -/// In case of errors during `git` command execution (e.g., in tarball sources), default values -/// are used to prevent panics. -pub fn generate_smart_stamp_hash( - builder: &Builder<'_>, - dir: &Path, - additional_input: &str, -) -> String { - let diff = helpers::git(Some(dir)) - .allow_failure() - .arg("diff") - .run_capture_stdout(builder) - .stdout_if_ok() - .unwrap_or_default(); - - let status = helpers::git(Some(dir)) - .allow_failure() - .arg("status") - .arg("--porcelain") - .arg("-z") - .arg("--untracked-files=normal") - .run_capture_stdout(builder) - .stdout_if_ok() - .unwrap_or_default(); - - let mut hasher = sha2::Sha256::new(); - - hasher.update(diff); - hasher.update(status); - hasher.update(additional_input); - - hex_encode(hasher.finalize().as_slice()) -} - -/// Ensures that the behavior dump directory is properly initialized. -pub fn prepare_behaviour_dump_dir(build: &Build) { - static INITIALIZED: OnceLock = OnceLock::new(); - - let dump_path = build.out.join("bootstrap-shims-dump"); - - let initialized = INITIALIZED.get().unwrap_or(&false); - if !initialized { - // clear old dumps - if dump_path.exists() { - t!(fs::remove_dir_all(&dump_path)); - } - - t!(fs::create_dir_all(&dump_path)); - - t!(INITIALIZED.set(true)); - } -} diff --git a/standalonex/src/src/utils/cache.rs b/standalonex/src/src/utils/cache.rs deleted file mode 100644 index 29342cc5..00000000 --- a/standalonex/src/src/utils/cache.rs +++ /dev/null @@ -1,257 +0,0 @@ -use std::any::{Any, TypeId}; -use std::borrow::Borrow; -use std::cell::RefCell; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::{LazyLock, Mutex}; -use std::{fmt, mem}; - -use crate::core::builder::Step; - -pub struct Interned(usize, PhantomData<*const T>); - -impl Default for Interned { - fn default() -> Self { - T::default().intern() - } -} - -impl Copy for Interned {} -impl Clone for Interned { - fn clone(&self) -> Interned { - *self - } -} - -impl PartialEq for Interned { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} -impl Eq for Interned {} - -impl PartialEq for Interned { - fn eq(&self, other: &str) -> bool { - *self == other - } -} -impl PartialEq<&str> for Interned { - fn eq(&self, other: &&str) -> bool { - **self == **other - } -} -impl PartialEq<&Interned> for Interned { - fn eq(&self, other: &&Self) -> bool { - self.0 == other.0 - } -} -impl PartialEq> for &Interned { - fn eq(&self, other: &Interned) -> bool { - self.0 == other.0 - } -} - -unsafe impl Send for Interned {} -unsafe impl Sync for Interned {} - -impl fmt::Display for Interned { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &str = self; - f.write_str(s) - } -} - -impl fmt::Debug for Interned -where - Self: Deref, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &U = self; - f.write_fmt(format_args!("{s:?}")) - } -} - -impl Hash for Interned { - fn hash(&self, state: &mut H) { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).hash(state) - } -} - -impl Deref for Interned { - type Target = T::Target; - fn deref(&self) -> &Self::Target { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&Self::Target, &Self::Target>(l.get(*self)) } - } -} - -impl, U: ?Sized> AsRef for Interned { - fn as_ref(&self) -> &U { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&U, &U>(l.get(*self).as_ref()) } - } -} - -impl PartialOrd for Interned { - fn partial_cmp(&self, other: &Self) -> Option { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).partial_cmp(l.get(*other)) - } -} - -impl Ord for Interned { - fn cmp(&self, other: &Self) -> Ordering { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).cmp(l.get(*other)) - } -} - -struct TyIntern { - items: Vec, - set: HashMap>, -} - -impl Default for TyIntern { - fn default() -> Self { - TyIntern { items: Vec::new(), set: Default::default() } - } -} - -impl TyIntern { - fn intern_borrow(&mut self, item: &B) -> Interned - where - B: Eq + Hash + ToOwned + ?Sized, - T: Borrow, - { - if let Some(i) = self.set.get(item) { - return *i; - } - let item = item.to_owned(); - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn intern(&mut self, item: T) -> Interned { - if let Some(i) = self.set.get(&item) { - return *i; - } - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn get(&self, i: Interned) -> &T { - &self.items[i.0] - } -} - -#[derive(Default)] -pub struct Interner { - strs: Mutex>, - paths: Mutex>, - lists: Mutex>>, -} - -trait Internable: Clone + Eq + Hash + 'static { - fn intern_cache() -> &'static Mutex>; - - fn intern(self) -> Interned { - Self::intern_cache().lock().unwrap().intern(self) - } -} - -impl Internable for String { - fn intern_cache() -> &'static Mutex> { - &INTERNER.strs - } -} - -impl Internable for PathBuf { - fn intern_cache() -> &'static Mutex> { - &INTERNER.paths - } -} - -impl Internable for Vec { - fn intern_cache() -> &'static Mutex> { - &INTERNER.lists - } -} - -impl Interner { - pub fn intern_str(&self, s: &str) -> Interned { - self.strs.lock().unwrap().intern_borrow(s) - } -} - -pub static INTERNER: LazyLock = LazyLock::new(Interner::default); - -/// This is essentially a `HashMap` which allows storing any type in its input and -/// any type in its output. It is a write-once cache; values are never evicted, -/// which means that references to the value can safely be returned from the -/// `get()` method. -#[derive(Debug)] -pub struct Cache( - RefCell< - HashMap< - TypeId, - Box, // actually a HashMap> - >, - >, -); - -impl Cache { - pub fn new() -> Cache { - Cache(RefCell::new(HashMap::new())) - } - - pub fn put(&self, step: S, value: S::Output) { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::>::default()) - .downcast_mut::>() - .expect("invalid type mapped"); - assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); - stepcache.insert(step, value); - } - - pub fn get(&self, step: &S) -> Option { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::>::default()) - .downcast_mut::>() - .expect("invalid type mapped"); - stepcache.get(step).cloned() - } -} - -#[cfg(test)] -impl Cache { - pub fn all(&mut self) -> Vec<(S, S::Output)> { - let cache = self.0.get_mut(); - let type_id = TypeId::of::(); - let mut v = cache - .remove(&type_id) - .map(|b| b.downcast::>().expect("correct type")) - .map(|m| m.into_iter().collect::>()) - .unwrap_or_default(); - v.sort_by_key(|(s, _)| s.clone()); - v - } - - pub fn contains(&self) -> bool { - self.0.borrow().contains_key(&TypeId::of::()) - } -} diff --git a/standalonex/src/src/utils/cc_detect.rs b/standalonex/src/src/utils/cc_detect.rs deleted file mode 100644 index 8cdb2744..00000000 --- a/standalonex/src/src/utils/cc_detect.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! C-compiler probing and detection. -//! -//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for -//! C and C++ compilers for each target configured. A compiler is found through -//! a number of vectors (in order of precedence) -//! -//! 1. Configuration via `target.$target.cc` in `config.toml`. -//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if -//! applicable -//! 3. Special logic to probe on OpenBSD -//! 4. The `CC_$target` environment variable. -//! 5. The `CC` environment variable. -//! 6. "cc" -//! -//! Some of this logic is implemented here, but much of it is farmed out to the -//! `cc` crate itself, so we end up having the same fallbacks as there. -//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is -//! used. -//! -//! It is intended that after this module has run no C/C++ compiler will -//! ever be probed for. Instead the compilers found here will be used for -//! everything. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::{env, iter}; - -use crate::core::config::TargetSelection; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::{Build, CLang, GitRepo}; - -// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, -// so use some simplified logic here. First we respect the environment variable `AR`, then -// try to infer the archiver path from the C compiler path. -// In the future this logic should be replaced by calling into the `cc` crate. -fn cc2ar(cc: &Path, target: TargetSelection) -> Option { - if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace('-', "_"))) { - Some(PathBuf::from(ar)) - } else if let Some(ar) = env::var_os("AR") { - Some(PathBuf::from(ar)) - } else if target.is_msvc() { - None - } else if target.contains("sbf") || target.contains("bpf") { - let parent = cc.parent().unwrap(); - let file = PathBuf::from("llvm-ar"); - Some(parent.join(file)) - } else if target.contains("musl") || target.contains("openbsd") { - Some(PathBuf::from("ar")) - } else if target.contains("vxworks") { - Some(PathBuf::from("wr-ar")) - } else if target.contains("android") || target.contains("-wasi") { - Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) - } else { - let parent = cc.parent().unwrap(); - let file = cc.file_name().unwrap().to_str().unwrap(); - for suffix in &["gcc", "cc", "clang"] { - if let Some(idx) = file.rfind(suffix) { - let mut file = file[..idx].to_owned(); - file.push_str("ar"); - return Some(parent.join(&file)); - } - } - Some(parent.join(file)) - } -} - -fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .opt_level(2) - .warnings(false) - .debug(false) - // Compress debuginfo - .flag_if_supported("-gz") - .target(&target.triple) - .host(&build.build.triple); - match build.crt_static(target) { - Some(a) => { - cfg.static_crt(a); - } - None => { - if target.is_msvc() { - cfg.static_crt(true); - } - if target.contains("musl") { - cfg.static_flag(true); - } - } - } - cfg -} - -pub fn find(build: &Build) { - let targets: HashSet<_> = match build.config.cmd { - // We don't need to check cross targets for these commands. - crate::Subcommand::Clean { .. } - | crate::Subcommand::Suggest { .. } - | crate::Subcommand::Format { .. } - | crate::Subcommand::Setup { .. } => { - build.hosts.iter().cloned().chain(iter::once(build.build)).collect() - } - - _ => { - // For all targets we're going to need a C compiler for building some shims - // and such as well as for being a linker for Rust code. - build - .targets - .iter() - .chain(&build.hosts) - .cloned() - .chain(iter::once(build.build)) - .collect() - } - }; - - for target in targets.into_iter() { - find_target(build, target); - } -} - -pub fn find_target(build: &Build, target: TargetSelection) { - let mut cfg = new_cc_build(build, target); - let config = build.config.target_config.get(&target); - if let Some(cc) = config - .and_then(|c| c.cc.clone()) - .or_else(|| default_compiler(&mut cfg, Language::C, target, build)) - { - cfg.compiler(cc); - } - - let compiler = cfg.get_compiler(); - let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { - ar - } else { - cc2ar(compiler.path(), target) - }; - - build.cc.borrow_mut().insert(target, compiler.clone()); - let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); - - // If we use llvm-libunwind, we will need a C++ compiler as well for all targets - // We'll need one anyways if the target triple is also a host triple - let mut cfg = new_cc_build(build, target); - cfg.cpp(true); - let cxx_configured = if let Some(cxx) = config - .and_then(|c| c.cxx.clone()) - .or_else(|| default_compiler(&mut cfg, Language::CPlusPlus, target, build)) - { - cfg.compiler(cxx); - true - } else { - // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). - cfg.try_get_compiler().is_ok() - }; - - // for VxWorks, record CXX compiler which will be used in lib.rs:linker() - if cxx_configured || target.contains("vxworks") { - let compiler = cfg.get_compiler(); - build.cxx.borrow_mut().insert(target, compiler); - } - - build.verbose(|| println!("CC_{} = {:?}", target.triple, build.cc(target))); - build.verbose(|| println!("CFLAGS_{} = {cflags:?}", target.triple)); - if let Ok(cxx) = build.cxx(target) { - let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); - build.verbose(|| println!("CXX_{} = {cxx:?}", target.triple)); - build.verbose(|| println!("CXXFLAGS_{} = {cxxflags:?}", target.triple)); - } - if let Some(ar) = ar { - build.verbose(|| println!("AR_{} = {ar:?}", target.triple)); - build.ar.borrow_mut().insert(target, ar); - } - - if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { - build.ranlib.borrow_mut().insert(target, ranlib); - } -} - -fn default_compiler( - cfg: &mut cc::Build, - compiler: Language, - target: TargetSelection, - build: &Build, -) -> Option { - match &*target.triple { - // When compiling for android we may have the NDK configured in the - // config.toml in which case we look there. Otherwise the default - // compiler already takes into account the triple in question. - t if t.contains("android") => { - build.config.android_ndk.as_ref().map(|ndk| ndk_compiler(compiler, &target.triple, ndk)) - } - - // The default gcc version from OpenBSD may be too old, try using egcc, - // which is a gcc version from ports, if this is the case. - t if t.contains("openbsd") => { - let c = cfg.get_compiler(); - let gnu_compiler = compiler.gcc(); - if !c.path().ends_with(gnu_compiler) { - return None; - } - - let mut cmd = BootstrapCommand::from(c.to_command()); - let output = cmd.arg("--version").run_capture_stdout(build).stdout(); - let i = output.find(" 4.")?; - match output[i + 3..].chars().next().unwrap() { - '0'..='6' => {} - _ => return None, - } - let alternative = format!("e{gnu_compiler}"); - if command(&alternative).run_capture(build).is_success() { - Some(PathBuf::from(alternative)) - } else { - None - } - } - - "mips-unknown-linux-musl" if compiler == Language::C => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - Some(PathBuf::from("mips-linux-musl-gcc")) - } else { - None - } - } - "mipsel-unknown-linux-musl" if compiler == Language::C => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - Some(PathBuf::from("mipsel-linux-musl-gcc")) - } else { - None - } - } - - "sbf-solana-solana" - | "sbpf-solana-solana" - | "sbpfv0-solana-solana" - | "sbpfv1-solana-solana" - | "sbpfv2-solana-solana" - | "sbpfv3-solana-solana" - | "sbpfv4-solana-solana" => { - Some(build.llvm_bin(target).join(compiler.clang())) - } - - t if t.contains("musl") && compiler == Language::C => { - if let Some(root) = build.musl_root(target) { - let guess = root.join("bin/musl-gcc"); - if guess.exists() { Some(guess) } else { None } - } else { - None - } - } - - t if t.contains("-wasi") => { - let root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?); - let compiler = match compiler { - Language::C => format!("{t}-clang"), - Language::CPlusPlus => format!("{t}-clang++"), - }; - let compiler = root.join("bin").join(compiler); - Some(compiler) - } - - _ => None, - } -} - -pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { - let mut triple_iter = triple.split('-'); - let triple_translated = if let Some(arch) = triple_iter.next() { - let arch_new = match arch { - "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", - other => other, - }; - std::iter::once(arch_new).chain(triple_iter).collect::>().join("-") - } else { - triple.to_string() - }; - - // The earliest API supported by NDK r26d is 21. - let api_level = "21"; - let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); - let host_tag = if cfg!(target_os = "macos") { - // The NDK uses universal binaries, so this is correct even on ARM. - "darwin-x86_64" - } else if cfg!(target_os = "windows") { - "windows-x86_64" - } else { - // NDK r26d only has official releases for macOS, Windows and Linux. - // Try the Linux directory everywhere else, on the assumption that the OS has an - // emulation layer that can cope (e.g. BSDs). - "linux-x86_64" - }; - ndk.join("toolchains").join("llvm").join("prebuilt").join(host_tag).join("bin").join(compiler) -} - -/// The target programming language for a native compiler. -#[derive(PartialEq)] -pub(crate) enum Language { - /// The compiler is targeting C. - C, - /// The compiler is targeting C++. - CPlusPlus, -} - -impl Language { - /// Obtains the name of a compiler in the GCC collection. - fn gcc(self) -> &'static str { - match self { - Language::C => "gcc", - Language::CPlusPlus => "g++", - } - } - - /// Obtains the name of a compiler in the clang suite. - fn clang(self) -> &'static str { - match self { - Language::C => "clang", - Language::CPlusPlus => "clang++", - } - } -} diff --git a/standalonex/src/src/utils/change_tracker.rs b/standalonex/src/src/utils/change_tracker.rs deleted file mode 100644 index 7f62ffb2..00000000 --- a/standalonex/src/src/utils/change_tracker.rs +++ /dev/null @@ -1,308 +0,0 @@ -//! This module facilitates the tracking system for major changes made to the bootstrap, -//! with the goal of keeping developers synchronized with important modifications in -//! the bootstrap. - -use std::fmt::Display; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Debug)] -pub struct ChangeInfo { - /// Represents the ID of PR caused major change on bootstrap. - pub change_id: usize, - pub severity: ChangeSeverity, - /// Provides a short summary of the change that will guide developers - /// on "how to handle/behave" in response to the changes. - pub summary: &'static str, -} - -#[derive(Clone, Debug)] -pub enum ChangeSeverity { - /// Used when build configurations continue working as before. - Info, - /// Used when the default value of an option changes, or support for an option is removed entirely, - /// potentially requiring developers to update their build configurations. - Warning, -} - -impl Display for ChangeSeverity { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ChangeSeverity::Info => write!(f, "INFO"), - ChangeSeverity::Warning => write!(f, "WARNING"), - } - } -} - -pub fn find_recent_config_change_ids(current_id: usize) -> Vec { - if !CONFIG_CHANGE_HISTORY.iter().any(|config| config.change_id == current_id) { - // If the current change-id is greater than the most recent one, return - // an empty list (it may be due to switching from a recent branch to an - // older one); otherwise, return the full list (assuming the user provided - // the incorrect change-id by accident). - if let Some(config) = CONFIG_CHANGE_HISTORY.iter().max_by_key(|config| config.change_id) { - if current_id > config.change_id { - return Vec::new(); - } - } - - return CONFIG_CHANGE_HISTORY.to_vec(); - } - - let index = - CONFIG_CHANGE_HISTORY.iter().position(|config| config.change_id == current_id).unwrap(); - - CONFIG_CHANGE_HISTORY - .iter() - .skip(index + 1) // Skip the current_id and IDs before it - .cloned() - .collect() -} - -pub fn human_readable_changes(changes: &[ChangeInfo]) -> String { - let mut message = String::new(); - - for change in changes { - message.push_str(&format!(" [{}] {}\n", change.severity, change.summary)); - message.push_str(&format!( - " - PR Link https://github.com/rust-lang/rust/pull/{}\n", - change.change_id - )); - } - - message -} - -/// Keeps track of major changes made to the bootstrap configuration. -/// -/// If you make any major changes (such as adding new values or changing default values), -/// please ensure adding `ChangeInfo` to the end(because the list must be sorted by the merge date) -/// of this list. -pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ - ChangeInfo { - change_id: 115898, - severity: ChangeSeverity::Info, - summary: "Implementation of this change-tracking system. Ignore this.", - }, - ChangeInfo { - change_id: 116998, - severity: ChangeSeverity::Info, - summary: "Removed android-ndk r15 support in favor of android-ndk r25b.", - }, - ChangeInfo { - change_id: 117435, - severity: ChangeSeverity::Info, - summary: "New option `rust.parallel-compiler` added to config.toml.", - }, - ChangeInfo { - change_id: 116881, - severity: ChangeSeverity::Warning, - summary: "Default value of `download-ci-llvm` was changed for `codegen` profile.", - }, - ChangeInfo { - change_id: 117813, - severity: ChangeSeverity::Info, - summary: "Use of the `if-available` value for `download-ci-llvm` is deprecated; prefer using the new `if-unchanged` value.", - }, - ChangeInfo { - change_id: 116278, - severity: ChangeSeverity::Info, - summary: "The `rust.use-lld` configuration now has different options ('external'/true or 'self-contained'), and its behaviour has changed.", - }, - ChangeInfo { - change_id: 118703, - severity: ChangeSeverity::Info, - summary: "Removed rust.run_dsymutil and dist.gpg_password_file config options, as they were unused.", - }, - ChangeInfo { - change_id: 119124, - severity: ChangeSeverity::Warning, - summary: "rust-analyzer-proc-macro-srv is no longer enabled by default. To build it, you must either enable it in the configuration or explicitly invoke it with x.py.", - }, - ChangeInfo { - change_id: 119373, - severity: ChangeSeverity::Info, - summary: "The dist.missing-tools config option was deprecated, as it was unused. If you are using it, remove it from your config, it will be removed soon.", - }, - ChangeInfo { - change_id: 102579, - severity: ChangeSeverity::Warning, - summary: "A new `optimized-compiler-builtins` option has been introduced. Whether to build llvm's `compiler-rt` from source is no longer implicitly controlled by git state. See the PR for more details.", - }, - ChangeInfo { - change_id: 120348, - severity: ChangeSeverity::Info, - summary: "New option `target..codegen-backends` added to config.toml.", - }, - ChangeInfo { - change_id: 121203, - severity: ChangeSeverity::Info, - summary: "A new `rust.frame-pointers` option has been introduced and made the default in the compiler and codegen profiles.", - }, - ChangeInfo { - change_id: 121278, - severity: ChangeSeverity::Warning, - summary: "The \"codegen\"/\"llvm\" profile has been removed and replaced with \"compiler\", use it instead for the same behavior.", - }, - ChangeInfo { - change_id: 118724, - severity: ChangeSeverity::Info, - summary: "`x install` now skips providing tarball sources (under 'build/dist' path) to speed up the installation process.", - }, - ChangeInfo { - change_id: 121976, - severity: ChangeSeverity::Info, - summary: "A new `boostrap-cache-path` option has been introduced which can be utilized to modify the cache path for bootstrap.", - }, - ChangeInfo { - change_id: 122108, - severity: ChangeSeverity::Info, - summary: "a new `target.*.runner` option is available to specify a wrapper executable required to run tests for a target", - }, - ChangeInfo { - change_id: 117458, - severity: ChangeSeverity::Info, - summary: "New option `rust.llvm-bitcode-linker` that will build the llvm-bitcode-linker.", - }, - ChangeInfo { - change_id: 121754, - severity: ChangeSeverity::Warning, - summary: "`rust.split-debuginfo` has been moved to `target..split-debuginfo` and its default value is determined for each target individually.", - }, - ChangeInfo { - change_id: 123711, - severity: ChangeSeverity::Warning, - summary: "The deprecated field `changelog-seen` has been removed. Using that field in `config.toml` from now on will result in breakage.", - }, - ChangeInfo { - change_id: 124501, - severity: ChangeSeverity::Info, - summary: "New option `build.lldb` that will override the default lldb binary path used in debuginfo tests", - }, - ChangeInfo { - change_id: 123337, - severity: ChangeSeverity::Info, - summary: r#"The compiler profile now defaults to rust.debuginfo-level = "line-tables-only""#, - }, - ChangeInfo { - change_id: 124129, - severity: ChangeSeverity::Warning, - summary: "`rust.lld` has a new default value of `true` on `x86_64-unknown-linux-gnu`. Starting at stage1, `rust-lld` will thus be this target's default linker. No config changes should be necessary.", - }, - ChangeInfo { - change_id: 125535, - severity: ChangeSeverity::Warning, - summary: "Removed `dist.missing-tools` configuration as it was deprecated long time ago.", - }, - ChangeInfo { - change_id: 126701, - severity: ChangeSeverity::Warning, - summary: "`llvm.lld` is enabled by default for the dist profile. If set to false, `lld` will not be included in the dist build.", - }, - ChangeInfo { - change_id: 127913, - severity: ChangeSeverity::Warning, - summary: "`debug-logging` option has been removed from the default `tools` profile.", - }, - ChangeInfo { - change_id: 127866, - severity: ChangeSeverity::Info, - summary: "the `wasm-component-ld` tool is now built as part of `build.extended` and can be a member of `build.tools`", - }, - ChangeInfo { - change_id: 120593, - severity: ChangeSeverity::Info, - summary: "Removed android-ndk r25b support in favor of android-ndk r26d.", - }, - ChangeInfo { - change_id: 125181, - severity: ChangeSeverity::Warning, - summary: "For tarball sources, default value for `rust.channel` will be taken from `src/ci/channel` file.", - }, - ChangeInfo { - change_id: 125642, - severity: ChangeSeverity::Info, - summary: "New option `llvm.libzstd` to control whether llvm is built with zstd support.", - }, - ChangeInfo { - change_id: 128841, - severity: ChangeSeverity::Warning, - summary: "./x test --rustc-args was renamed to --compiletest-rustc-args as it only applies there. ./x miri --rustc-args was also removed.", - }, - ChangeInfo { - change_id: 129295, - severity: ChangeSeverity::Info, - summary: "The `build.profiler` option now tries to use source code from `download-ci-llvm` if possible, instead of checking out the `src/llvm-project` submodule.", - }, - ChangeInfo { - change_id: 129152, - severity: ChangeSeverity::Info, - summary: "New option `build.cargo-clippy` added for supporting the use of custom/external clippy.", - }, - ChangeInfo { - change_id: 129925, - severity: ChangeSeverity::Warning, - summary: "Removed `rust.split-debuginfo` as it was deprecated long time ago.", - }, - ChangeInfo { - change_id: 129176, - severity: ChangeSeverity::Info, - summary: "New option `llvm.enzyme` to control whether the llvm based autodiff tool (Enzyme) is built.", - }, - ChangeInfo { - change_id: 129473, - severity: ChangeSeverity::Warning, - summary: "`download-ci-llvm = true` now checks if CI llvm is available and has become the default for the compiler profile", - }, - ChangeInfo { - change_id: 130202, - severity: ChangeSeverity::Info, - summary: "'tools' and 'library' profiles switched `download-ci-llvm` option from `if-unchanged` to `true`.", - }, - ChangeInfo { - change_id: 130110, - severity: ChangeSeverity::Info, - summary: "New option `dist.vendor` added to control whether bootstrap should vendor dependencies for dist tarball.", - }, - ChangeInfo { - change_id: 130529, - severity: ChangeSeverity::Info, - summary: "If `llvm.download-ci-llvm` is not defined, it defaults to `true`.", - }, - ChangeInfo { - change_id: 131075, - severity: ChangeSeverity::Info, - summary: "New option `./x setup editor` added, replacing `./x setup vscode` and adding support for vim, emacs and helix.", - }, - ChangeInfo { - change_id: 131838, - severity: ChangeSeverity::Info, - summary: "Allow setting `--jobs` in config.toml with `build.jobs`.", - }, - ChangeInfo { - change_id: 131181, - severity: ChangeSeverity::Info, - summary: "New option `build.compiletest-diff-tool` that adds support for a custom differ for compiletest", - }, - ChangeInfo { - change_id: 131513, - severity: ChangeSeverity::Info, - summary: "New option `llvm.offload` to control whether the llvm offload runtime for GPU support is built. Implicitly enables the openmp runtime as dependency.", - }, - ChangeInfo { - change_id: 132282, - severity: ChangeSeverity::Warning, - summary: "Deprecated `rust.parallel_compiler` as the compiler now always defaults to being parallel (with 1 thread)", - }, - ChangeInfo { - change_id: 132494, - severity: ChangeSeverity::Info, - summary: "`download-rustc='if-unchanged'` is now a default option for library profile.", - }, - ChangeInfo { - change_id: 133207, - severity: ChangeSeverity::Info, - summary: "`rust.llvm-tools` is now enabled by default when no `config.toml` is provided.", - }, -]; diff --git a/standalonex/src/src/utils/change_tracker/tests.rs b/standalonex/src/src/utils/change_tracker/tests.rs deleted file mode 100644 index 730b65b4..00000000 --- a/standalonex/src/src/utils/change_tracker/tests.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::{CONFIG_CHANGE_HISTORY, find_recent_config_change_ids}; - -#[test] -fn test_find_recent_config_change_ids() { - // If change-id is greater than the most recent one, result should be empty. - assert!(find_recent_config_change_ids(usize::MAX).is_empty()); - - // There is no change-id equal to or less than 0, result should include the entire change history. - assert_eq!(find_recent_config_change_ids(0).len(), CONFIG_CHANGE_HISTORY.len()); -} diff --git a/standalonex/src/src/utils/channel.rs b/standalonex/src/src/utils/channel.rs deleted file mode 100644 index 4a9ecc7a..00000000 --- a/standalonex/src/src/utils/channel.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Build configuration for Rust's release channels. -//! -//! Implements the stable/beta/nightly channel distinctions by setting various -//! flags like the `unstable_features`, calculating variables like `release` and -//! `package_vers`, and otherwise indicating to the compiler what it should -//! print out as part of its version information. - -use std::fs; -use std::path::Path; - -use super::helpers; -use crate::Build; -use crate::utils::helpers::{start_process, t}; - -#[derive(Clone, Default)] -pub enum GitInfo { - /// This is not a git repository. - #[default] - Absent, - /// This is a git repository. - /// If the info should be used (`omit_git_hash` is false), this will be - /// `Some`, otherwise it will be `None`. - Present(Option), - /// This is not a git repository, but the info can be fetched from the - /// `git-commit-info` file. - RecordedForTarball(Info), -} - -#[derive(Clone)] -pub struct Info { - pub commit_date: String, - pub sha: String, - pub short_sha: String, -} - -impl GitInfo { - pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { - // See if this even begins to look like a git dir - if !dir.join(".git").exists() { - match read_commit_info_file(dir) { - Some(info) => return GitInfo::RecordedForTarball(info), - None => return GitInfo::Absent, - } - } - - // Make sure git commands work - match helpers::git(Some(dir)).arg("rev-parse").as_command_mut().output() { - Ok(ref out) if out.status.success() => {} - _ => return GitInfo::Absent, - } - - // If we're ignoring the git info, we don't actually need to collect it, just make sure this - // was a git repo in the first place. - if omit_git_hash { - return GitInfo::Present(None); - } - - // Ok, let's scrape some info - let ver_date = start_process( - helpers::git(Some(dir)) - .arg("log") - .arg("-1") - .arg("--date=short") - .arg("--pretty=format:%cd") - .as_command_mut(), - ); - let ver_hash = - start_process(helpers::git(Some(dir)).arg("rev-parse").arg("HEAD").as_command_mut()); - let short_ver_hash = start_process( - helpers::git(Some(dir)).arg("rev-parse").arg("--short=9").arg("HEAD").as_command_mut(), - ); - GitInfo::Present(Some(Info { - commit_date: ver_date().trim().to_string(), - sha: ver_hash().trim().to_string(), - short_sha: short_ver_hash().trim().to_string(), - })) - } - - pub fn info(&self) -> Option<&Info> { - match self { - GitInfo::Absent => None, - GitInfo::Present(info) => info.as_ref(), - GitInfo::RecordedForTarball(info) => Some(info), - } - } - - pub fn sha(&self) -> Option<&str> { - self.info().map(|s| &s.sha[..]) - } - - pub fn sha_short(&self) -> Option<&str> { - self.info().map(|s| &s.short_sha[..]) - } - - pub fn commit_date(&self) -> Option<&str> { - self.info().map(|s| &s.commit_date[..]) - } - - pub fn version(&self, build: &Build, num: &str) -> String { - let mut version = build.release(num); - if let Some(inner) = self.info() { - version.push_str(" ("); - version.push_str(&inner.short_sha); - version.push(' '); - version.push_str(&inner.commit_date); - version.push(')'); - } - version - } - - /// Returns whether this directory has a `.git` directory which should be managed by bootstrap. - pub fn is_managed_git_subrepository(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::RecordedForTarball(_) => false, - GitInfo::Present(_) => true, - } - } - - /// Returns whether this is being built from a tarball. - pub fn is_from_tarball(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::Present(_) => false, - GitInfo::RecordedForTarball(_) => true, - } - } -} - -/// Read the commit information from the `git-commit-info` file given the -/// project root. -pub fn read_commit_info_file(root: &Path) -> Option { - if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) { - let mut lines = contents.lines(); - let sha = lines.next(); - let short_sha = lines.next(); - let commit_date = lines.next(); - let info = match (commit_date, sha, short_sha) { - (Some(commit_date), Some(sha), Some(short_sha)) => Info { - commit_date: commit_date.to_owned(), - sha: sha.to_owned(), - short_sha: short_sha.to_owned(), - }, - _ => panic!("the `git-commit-info` file is malformed"), - }; - Some(info) - } else { - None - } -} - -/// Write the commit information to the `git-commit-info` file given the project -/// root. -pub fn write_commit_info_file(root: &Path, info: &Info) { - let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date); - t!(fs::write(root.join("git-commit-info"), commit_info)); -} - -/// Write the commit hash to the `git-commit-hash` file given the project root. -pub fn write_commit_hash_file(root: &Path, sha: &str) { - t!(fs::write(root.join("git-commit-hash"), sha)); -} diff --git a/standalonex/src/src/utils/exec.rs b/standalonex/src/src/utils/exec.rs deleted file mode 100644 index 530d760a..00000000 --- a/standalonex/src/src/utils/exec.rs +++ /dev/null @@ -1,326 +0,0 @@ -use std::ffi::OsStr; -use std::fmt::{Debug, Formatter}; -use std::path::Path; -use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; - -use build_helper::ci::CiEnv; -use build_helper::drop_bomb::DropBomb; - -use crate::Build; - -/// What should be done when the command fails. -#[derive(Debug, Copy, Clone)] -pub enum BehaviorOnFailure { - /// Immediately stop bootstrap. - Exit, - /// Delay failure until the end of bootstrap invocation. - DelayFail, - /// Ignore the failure, the command can fail in an expected way. - Ignore, -} - -/// How should the output of a specific stream of the command (stdout/stderr) be handled -/// (whether it should be captured or printed). -#[derive(Debug, Copy, Clone)] -pub enum OutputMode { - /// Prints the stream by inheriting it from the bootstrap process. - Print, - /// Captures the stream into memory. - Capture, -} - -impl OutputMode { - pub fn captures(&self) -> bool { - match self { - OutputMode::Print => false, - OutputMode::Capture => true, - } - } - - pub fn stdio(&self) -> Stdio { - match self { - OutputMode::Print => Stdio::inherit(), - OutputMode::Capture => Stdio::piped(), - } - } -} - -/// Wrapper around `std::process::Command`. -/// -/// By default, the command will exit bootstrap if it fails. -/// If you want to allow failures, use [allow_failure]. -/// If you want to delay failures until the end of bootstrap, use [delay_failure]. -/// -/// By default, the command will print its stdout/stderr to stdout/stderr of bootstrap ([OutputMode::Print]). -/// If you want to handle the output programmatically, use [BootstrapCommand::run_capture]. -/// -/// Bootstrap will print a debug log to stdout if the command fails and failure is not allowed. -/// -/// [allow_failure]: BootstrapCommand::allow_failure -/// [delay_failure]: BootstrapCommand::delay_failure -pub struct BootstrapCommand { - command: Command, - pub failure_behavior: BehaviorOnFailure, - // Run the command even during dry run - pub run_always: bool, - // This field makes sure that each command is executed (or disarmed) before it is dropped, - // to avoid forgetting to execute a command. - drop_bomb: DropBomb, -} - -impl BootstrapCommand { - #[track_caller] - pub fn new>(program: S) -> Self { - Command::new(program).into() - } - - pub fn arg>(&mut self, arg: S) -> &mut Self { - self.command.arg(arg.as_ref()); - self - } - - pub fn args(&mut self, args: I) -> &mut Self - where - I: IntoIterator, - S: AsRef, - { - self.command.args(args); - self - } - - pub fn env(&mut self, key: K, val: V) -> &mut Self - where - K: AsRef, - V: AsRef, - { - self.command.env(key, val); - self - } - - pub fn get_envs(&self) -> CommandEnvs<'_> { - self.command.get_envs() - } - - pub fn get_args(&self) -> CommandArgs<'_> { - self.command.get_args() - } - - pub fn env_remove>(&mut self, key: K) -> &mut Self { - self.command.env_remove(key); - self - } - - pub fn current_dir>(&mut self, dir: P) -> &mut Self { - self.command.current_dir(dir); - self - } - - #[must_use] - pub fn delay_failure(self) -> Self { - Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } - } - - #[must_use] - pub fn fail_fast(self) -> Self { - Self { failure_behavior: BehaviorOnFailure::Exit, ..self } - } - - #[must_use] - pub fn allow_failure(self) -> Self { - Self { failure_behavior: BehaviorOnFailure::Ignore, ..self } - } - - pub fn run_always(&mut self) -> &mut Self { - self.run_always = true; - self - } - - /// Run the command, while printing stdout and stderr. - /// Returns true if the command has succeeded. - #[track_caller] - pub fn run(&mut self, builder: &Build) -> bool { - builder.run(self, OutputMode::Print, OutputMode::Print).is_success() - } - - /// Run the command, while capturing and returning all its output. - #[track_caller] - pub fn run_capture(&mut self, builder: &Build) -> CommandOutput { - builder.run(self, OutputMode::Capture, OutputMode::Capture) - } - - /// Run the command, while capturing and returning stdout, and printing stderr. - #[track_caller] - pub fn run_capture_stdout(&mut self, builder: &Build) -> CommandOutput { - builder.run(self, OutputMode::Capture, OutputMode::Print) - } - - /// Provides access to the stdlib Command inside. - /// FIXME: This function should be eventually removed from bootstrap. - pub fn as_command_mut(&mut self) -> &mut Command { - // We don't know what will happen with the returned command, so we need to mark this - // command as executed proactively. - self.mark_as_executed(); - &mut self.command - } - - /// Mark the command as being executed, disarming the drop bomb. - /// If this method is not called before the command is dropped, its drop will panic. - pub fn mark_as_executed(&mut self) { - self.drop_bomb.defuse(); - } - - /// Returns the source code location where this command was created. - pub fn get_created_location(&self) -> std::panic::Location<'static> { - self.drop_bomb.get_created_location() - } - - /// If in a CI environment, forces the command to run with colors. - pub fn force_coloring_in_ci(&mut self) { - if CiEnv::is_ci() { - // Due to use of stamp/docker, the output stream of bootstrap is not - // a TTY in CI, so coloring is by-default turned off. - // The explicit `TERM=xterm` environment is needed for - // `--color always` to actually work. This env var was lost when - // compiling through the Makefile. Very strange. - self.env("TERM", "xterm").args(["--color", "always"]); - } - } -} - -impl Debug for BootstrapCommand { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.command)?; - write!(f, " (failure_mode={:?})", self.failure_behavior) - } -} - -impl From for BootstrapCommand { - #[track_caller] - fn from(command: Command) -> Self { - let program = command.get_program().to_owned(); - - Self { - command, - failure_behavior: BehaviorOnFailure::Exit, - run_always: false, - drop_bomb: DropBomb::arm(program), - } - } -} - -/// Represents the current status of `BootstrapCommand`. -enum CommandStatus { - /// The command has started and finished with some status. - Finished(ExitStatus), - /// It was not even possible to start the command. - DidNotStart, -} - -/// Create a new BootstrapCommand. This is a helper function to make command creation -/// shorter than `BootstrapCommand::new`. -#[track_caller] -#[must_use] -pub fn command>(program: S) -> BootstrapCommand { - BootstrapCommand::new(program) -} - -/// Represents the output of an executed process. -pub struct CommandOutput { - status: CommandStatus, - stdout: Option>, - stderr: Option>, -} - -impl CommandOutput { - #[must_use] - pub fn did_not_start(stdout: OutputMode, stderr: OutputMode) -> Self { - Self { - status: CommandStatus::DidNotStart, - stdout: match stdout { - OutputMode::Print => None, - OutputMode::Capture => Some(vec![]), - }, - stderr: match stderr { - OutputMode::Print => None, - OutputMode::Capture => Some(vec![]), - }, - } - } - - #[must_use] - pub fn from_output(output: Output, stdout: OutputMode, stderr: OutputMode) -> Self { - Self { - status: CommandStatus::Finished(output.status), - stdout: match stdout { - OutputMode::Print => None, - OutputMode::Capture => Some(output.stdout), - }, - stderr: match stderr { - OutputMode::Print => None, - OutputMode::Capture => Some(output.stderr), - }, - } - } - - #[must_use] - pub fn is_success(&self) -> bool { - match self.status { - CommandStatus::Finished(status) => status.success(), - CommandStatus::DidNotStart => false, - } - } - - #[must_use] - pub fn is_failure(&self) -> bool { - !self.is_success() - } - - #[must_use] - pub fn status(&self) -> Option { - match self.status { - CommandStatus::Finished(status) => Some(status), - CommandStatus::DidNotStart => None, - } - } - - #[must_use] - pub fn stdout(&self) -> String { - String::from_utf8( - self.stdout.clone().expect("Accessing stdout of a command that did not capture stdout"), - ) - .expect("Cannot parse process stdout as UTF-8") - } - - #[must_use] - pub fn stdout_if_present(&self) -> Option { - self.stdout.as_ref().and_then(|s| String::from_utf8(s.clone()).ok()) - } - - #[must_use] - pub fn stdout_if_ok(&self) -> Option { - if self.is_success() { Some(self.stdout()) } else { None } - } - - #[must_use] - pub fn stderr(&self) -> String { - String::from_utf8( - self.stderr.clone().expect("Accessing stderr of a command that did not capture stderr"), - ) - .expect("Cannot parse process stderr as UTF-8") - } - - #[must_use] - pub fn stderr_if_present(&self) -> Option { - self.stderr.as_ref().and_then(|s| String::from_utf8(s.clone()).ok()) - } -} - -impl Default for CommandOutput { - fn default() -> Self { - Self { - status: CommandStatus::Finished(ExitStatus::default()), - stdout: Some(vec![]), - stderr: Some(vec![]), - } - } -} diff --git a/standalonex/src/src/utils/helpers.rs b/standalonex/src/src/utils/helpers.rs deleted file mode 100644 index c226d07d..00000000 --- a/standalonex/src/src/utils/helpers.rs +++ /dev/null @@ -1,621 +0,0 @@ -//! Various utility functions used throughout bootstrap. -//! -//! Simple things like testing the various filesystem operations here and there, -//! not a lot of interesting happenings here unfortunately. - -use std::ffi::OsStr; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::OnceLock; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use std::{env, fs, io, str}; - -use build_helper::util::fail; -use object::read::archive::ArchiveFile; - -use crate::LldMode; -use crate::core::builder::Builder; -use crate::core::config::{Config, TargetSelection}; -pub use crate::utils::shared_helpers::{dylib_path, dylib_path_var}; - -#[cfg(test)] -mod tests; - -/// A helper macro to `unwrap` a result except also print out details like: -/// -/// * The file/line of the panic -/// * The expression that failed -/// * The error itself -/// -/// This is currently used judiciously throughout the build system rather than -/// using a `Result` with `try!`, but this may change one day... -#[macro_export] -macro_rules! t { - ($e:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - } - }; - // it can show extra info in the second parameter - ($e:expr, $extra:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), - } - }; -} -pub use t; - -use crate::utils::exec::{BootstrapCommand, command}; - -pub fn exe(name: &str, target: TargetSelection) -> String { - crate::utils::shared_helpers::exe(name, &target.triple) -} - -/// Returns `true` if the file name given looks like a dynamic library. -pub fn is_dylib(path: &Path) -> bool { - path.extension().and_then(|ext| ext.to_str()).map_or(false, |ext| { - ext == "dylib" || ext == "so" || ext == "dll" || (ext == "a" && is_aix_shared_archive(path)) - }) -} - -fn is_aix_shared_archive(path: &Path) -> bool { - // FIXME(#133268): reading the entire file as &[u8] into memory seems excessive - // look into either mmap it or use the ReadCache - let data = match fs::read(path) { - Ok(data) => data, - Err(_) => return false, - }; - let file = match ArchiveFile::parse(&*data) { - Ok(file) => file, - Err(_) => return false, - }; - - file.members() - .filter_map(Result::ok) - .any(|entry| String::from_utf8_lossy(entry.name()).contains(".so")) -} - -/// Returns `true` if the file name given looks like a debug info file -pub fn is_debug_info(name: &str) -> bool { - // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) - name.ends_with(".pdb") -} - -/// Returns the corresponding relative library directory that the compiler's -/// dylibs will be found in. -pub fn libdir(target: TargetSelection) -> &'static str { - if target.is_windows() { "bin" } else { "lib" } -} - -/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. -/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! -pub fn add_dylib_path(path: Vec, cmd: &mut BootstrapCommand) { - let mut list = dylib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(dylib_path_var(), t!(env::join_paths(list))); -} - -/// Adds a list of lookup paths to `cmd`'s link library lookup path. -pub fn add_link_lib_path(path: Vec, cmd: &mut BootstrapCommand) { - let mut list = link_lib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(link_lib_path_var(), t!(env::join_paths(list))); -} - -/// Returns the environment variable which the link library lookup path -/// resides in for this platform. -fn link_lib_path_var() -> &'static str { - if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } -} - -/// Parses the `link_lib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -fn link_lib_path() -> Vec { - let var = match env::var_os(link_lib_path_var()) { - Some(v) => v, - None => return vec![], - }; - env::split_paths(&var).collect() -} - -pub struct TimeIt(bool, Instant); - -/// Returns an RAII structure that prints out how long it took to drop. -pub fn timeit(builder: &Builder<'_>) -> TimeIt { - TimeIt(builder.config.dry_run(), Instant::now()) -} - -impl Drop for TimeIt { - fn drop(&mut self) { - let time = self.1.elapsed(); - if !self.0 { - println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); - } - } -} - -/// Used for download caching -pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key -} - -/// Symlinks two directories, using junctions on Windows and normal symlinks on -/// Unix. -pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { - if config.dry_run() { - return Ok(()); - } - let _ = fs::remove_dir_all(link); - return symlink_dir_inner(original, link); - - #[cfg(not(windows))] - fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> { - use std::os::unix::fs; - fs::symlink(original, link) - } - - #[cfg(windows)] - fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { - junction::create(target, junction) - } -} - -/// Rename a file if from and to are in the same filesystem or -/// copy and remove the file otherwise -pub fn move_file, Q: AsRef>(from: P, to: Q) -> io::Result<()> { - match fs::rename(&from, &to) { - // FIXME: Once `ErrorKind::CrossesDevices` is stabilized use - // if e.kind() == io::ErrorKind::CrossesDevices { - #[cfg(unix)] - Err(e) if e.raw_os_error() == Some(libc::EXDEV) => { - std::fs::copy(&from, &to)?; - std::fs::remove_file(&from) - } - r => r, - } -} - -pub fn forcing_clang_based_tests() -> bool { - if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { - match &var.to_string_lossy().to_lowercase()[..] { - "1" | "yes" | "on" => true, - "0" | "no" | "off" => false, - other => { - // Let's make sure typos don't go unnoticed - panic!( - "Unrecognized option '{other}' set in \ - RUSTBUILD_FORCE_CLANG_BASED_TESTS" - ) - } - } - } else { - false - } -} - -pub fn use_host_linker(target: TargetSelection) -> bool { - // FIXME: this information should be gotten by checking the linker flavor - // of the rustc target - !(target.contains("emscripten") - || target.contains("wasm32") - || target.contains("nvptx") - || target.contains("fortanix") - || target.contains("fuchsia") - || target.contains("bpf") - || target.contains("sbf") - || target.contains("switch")) -} - -pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool { - if target.contains("linux") { - target.contains("x86_64") - || target.contains("aarch64") - || target.contains("s390x") - || target.contains("riscv64gc") - } else if target.contains("darwin") { - target.contains("x86_64") || target.contains("aarch64") - } else if target.is_windows() { - target.contains("x86_64") - } else { - false - } -} - -pub fn is_valid_test_suite_arg<'a, P: AsRef>( - path: &'a Path, - suite_path: P, - builder: &Builder<'_>, -) -> Option<&'a str> { - let suite_path = suite_path.as_ref(); - let path = match path.strip_prefix(".") { - Ok(p) => p, - Err(_) => path, - }; - if !path.starts_with(suite_path) { - return None; - } - let abs_path = builder.src.join(path); - let exists = abs_path.is_dir() || abs_path.is_file(); - if !exists { - panic!( - "Invalid test suite filter \"{}\": file or directory does not exist", - abs_path.display() - ); - } - // Since test suite paths are themselves directories, if we don't - // specify a directory or file, we'll get an empty string here - // (the result of the test suite directory without its suite prefix). - // Therefore, we need to filter these out, as only the first --test-args - // flag is respected, so providing an empty --test-args conflicts with - // any following it. - match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { - Some(s) if !s.is_empty() => Some(s), - _ => None, - } -} - -// FIXME: get rid of this function -pub fn check_run(cmd: &mut BootstrapCommand, print_cmd_on_fail: bool) -> bool { - let status = match cmd.as_command_mut().status() { - Ok(status) => status, - Err(e) => { - println!("failed to execute command: {cmd:?}\nERROR: {e}"); - return false; - } - }; - if !status.success() && print_cmd_on_fail { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {status}\n\n" - ); - } - status.success() -} - -pub fn make(host: &str) -> PathBuf { - if host.contains("dragonfly") - || host.contains("freebsd") - || host.contains("netbsd") - || host.contains("openbsd") - { - PathBuf::from("gmake") - } else { - PathBuf::from("make") - } -} - -#[track_caller] -pub fn output(cmd: &mut Command) -> String { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), - }; - if !output.status.success() { - panic!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}", - cmd, output.status - ); - } - String::from_utf8(output.stdout).unwrap() -} - -/// Spawn a process and return a closure that will wait for the process -/// to finish and then return its output. This allows the spawned process -/// to do work without immediately blocking bootstrap. -#[track_caller] -pub fn start_process(cmd: &mut Command) -> impl FnOnce() -> String { - let child = match cmd.stderr(Stdio::inherit()).stdout(Stdio::piped()).spawn() { - Ok(child) => child, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), - }; - - let command = format!("{:?}", cmd); - - move || { - let output = child.wait_with_output().unwrap(); - - if !output.status.success() { - panic!( - "command did not execute successfully: {}\n\ - expected success, got: {}", - command, output.status - ); - } - - String::from_utf8(output.stdout).unwrap() - } -} - -/// Returns the last-modified time for `path`, or zero if it doesn't exist. -pub fn mtime(path: &Path) -> SystemTime { - fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) -} - -/// Returns `true` if `dst` is up to date given that the file or files in `src` -/// are used to generate it. -/// -/// Uses last-modified time checks to verify this. -pub fn up_to_date(src: &Path, dst: &Path) -> bool { - if !dst.exists() { - return false; - } - let threshold = mtime(dst); - let meta = match fs::metadata(src) { - Ok(meta) => meta, - Err(e) => panic!("source {src:?} failed to get metadata: {e}"), - }; - if meta.is_dir() { - dir_up_to_date(src, threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) <= threshold - } -} - -/// Returns the filename without the hash prefix added by the cc crate. -/// -/// Since v1.0.78 of the cc crate, object files are prefixed with a 16-character hash -/// to avoid filename collisions. -pub fn unhashed_basename(obj: &Path) -> &str { - let basename = obj.file_stem().unwrap().to_str().expect("UTF-8 file name"); - basename.split_once('-').unwrap().1 -} - -fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { - t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { - let meta = t!(e.metadata()); - if meta.is_dir() { - dir_up_to_date(&e.path(), threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) < threshold - } - }) -} - -/// Adapted from -/// -/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource -/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime -/// missing. This function returns the path to that directory. -pub fn get_clang_cl_resource_dir(builder: &Builder<'_>, clang_cl_path: &str) -> PathBuf { - // Similar to how LLVM does it, to find clang's library runtime directory: - // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. - let mut builtins_locator = command(clang_cl_path); - builtins_locator.args(["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); - - let clang_rt_builtins = builtins_locator.run_capture_stdout(builder).stdout(); - let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); - assert!( - clang_rt_builtins.exists(), - "`clang-cl` must correctly locate the library runtime directory" - ); - - // - the profiler runtime will be located in the same directory as the builtins lib, like - // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. - let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); - clang_rt_dir.to_path_buf() -} - -/// Returns a flag that configures LLD to use only a single thread. -/// If we use an external LLD, we need to find out which version is it to know which flag should we -/// pass to it (LLD older than version 10 had a different flag). -fn lld_flag_no_threads(builder: &Builder<'_>, lld_mode: LldMode, is_windows: bool) -> &'static str { - static LLD_NO_THREADS: OnceLock<(&'static str, &'static str)> = OnceLock::new(); - - let new_flags = ("/threads:1", "--threads=1"); - let old_flags = ("/no-threads", "--no-threads"); - - let (windows_flag, other_flag) = LLD_NO_THREADS.get_or_init(|| { - let newer_version = match lld_mode { - LldMode::External => { - let mut cmd = command("lld"); - cmd.arg("-flavor").arg("ld").arg("--version"); - let out = cmd.run_capture_stdout(builder).stdout(); - match (out.find(char::is_numeric), out.find('.')) { - (Some(b), Some(e)) => out.as_str()[b..e].parse::().ok().unwrap_or(14) > 10, - _ => true, - } - } - _ => true, - }; - if newer_version { new_flags } else { old_flags } - }); - if is_windows { windows_flag } else { other_flag } -} - -pub fn dir_is_empty(dir: &Path) -> bool { - t!(std::fs::read_dir(dir)).next().is_none() -} - -/// Extract the beta revision from the full version string. -/// -/// The full version string looks like "a.b.c-beta.y". And we need to extract -/// the "y" part from the string. -pub fn extract_beta_rev(version: &str) -> Option { - let parts = version.splitn(2, "-beta.").collect::>(); - let count = parts.get(1).and_then(|s| s.find(' ').map(|p| s[..p].to_string())); - - count -} - -pub enum LldThreads { - Yes, - No, -} - -/// Returns the linker arguments for rustc/rustdoc for the given builder and target. -pub fn linker_args( - builder: &Builder<'_>, - target: TargetSelection, - lld_threads: LldThreads, -) -> Vec { - let mut args = linker_flags(builder, target, lld_threads); - - if let Some(linker) = builder.linker(target) { - args.push(format!("-Clinker={}", linker.display())); - } - - args -} - -/// Returns the linker arguments for rustc/rustdoc for the given builder and target, without the -/// -Clinker flag. -pub fn linker_flags( - builder: &Builder<'_>, - target: TargetSelection, - lld_threads: LldThreads, -) -> Vec { - let mut args = vec![]; - if !builder.is_lld_direct_linker(target) && builder.config.lld_mode.is_used() { - args.push(String::from("-Clink-arg=-fuse-ld=lld")); - - if matches!(lld_threads, LldThreads::No) { - args.push(format!( - "-Clink-arg=-Wl,{}", - lld_flag_no_threads(builder, builder.config.lld_mode, target.is_windows()) - )); - } - } - args -} - -pub fn add_rustdoc_cargo_linker_args( - cmd: &mut BootstrapCommand, - builder: &Builder<'_>, - target: TargetSelection, - lld_threads: LldThreads, -) { - let args = linker_args(builder, target, lld_threads); - let mut flags = cmd - .get_envs() - .find_map(|(k, v)| if k == OsStr::new("RUSTDOCFLAGS") { v } else { None }) - .unwrap_or_default() - .to_os_string(); - for arg in args { - if !flags.is_empty() { - flags.push(" "); - } - flags.push(arg); - } - if !flags.is_empty() { - cmd.env("RUSTDOCFLAGS", flags); - } -} - -/// Converts `T` into a hexadecimal `String`. -pub fn hex_encode(input: T) -> String -where - T: AsRef<[u8]>, -{ - use std::fmt::Write; - - input.as_ref().iter().fold(String::with_capacity(input.as_ref().len() * 2), |mut acc, &byte| { - write!(&mut acc, "{:02x}", byte).expect("Failed to write byte to the hex String."); - acc - }) -} - -/// Create a `--check-cfg` argument invocation for a given name -/// and it's values. -pub fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String { - // Creating a string of the values by concatenating each value: - // ',values("tvos","watchos")' or '' (nothing) when there are no values. - let next = match values { - Some(values) => { - let mut tmp = values.iter().flat_map(|val| [",", "\"", val, "\""]).collect::(); - - tmp.insert_str(1, "values("); - tmp.push(')'); - tmp - } - None => "".to_string(), - }; - format!("--check-cfg=cfg({name}{next})") -} - -/// Prepares `BootstrapCommand` that runs git inside the source directory if given. -/// -/// Whenever a git invocation is needed, this function should be preferred over -/// manually building a git `BootstrapCommand`. This approach allows us to manage -/// bootstrap-specific needs/hacks from a single source, rather than applying them on next to every -/// git command creation, which is painful to ensure that the required change is applied -/// on each one of them correctly. -#[track_caller] -pub fn git(source_dir: Option<&Path>) -> BootstrapCommand { - let mut git = command("git"); - - if let Some(source_dir) = source_dir { - git.current_dir(source_dir); - // If we are running inside git (e.g. via a hook), `GIT_DIR` is set and takes precedence - // over the current dir. Un-set it to make the current dir matter. - git.env_remove("GIT_DIR"); - // Also un-set some other variables, to be on the safe side (based on cargo's - // `fetch_with_cli`). In particular un-setting `GIT_INDEX_FILE` is required to fix some odd - // misbehavior. - git.env_remove("GIT_WORK_TREE") - .env_remove("GIT_INDEX_FILE") - .env_remove("GIT_OBJECT_DIRECTORY") - .env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES"); - } - - git -} - -/// Sets the file times for a given file at `path`. -pub fn set_file_times>(path: P, times: fs::FileTimes) -> io::Result<()> { - // Windows requires file to be writable to modify file times. But on Linux CI the file does not - // need to be writable to modify file times and might be read-only. - let f = if cfg!(windows) { - fs::File::options().write(true).open(path)? - } else { - fs::File::open(path)? - }; - f.set_times(times) -} - -pub struct HashStamp { - pub path: PathBuf, - pub hash: Option>, -} - -impl HashStamp { - pub fn new(path: PathBuf, hash: Option<&str>) -> Self { - HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } - } - - pub fn is_done(&self) -> bool { - match fs::read(&self.path) { - Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), - Err(e) if e.kind() == io::ErrorKind::NotFound => false, - Err(e) => { - panic!("failed to read stamp file `{}`: {}", self.path.display(), e); - } - } - } - - pub fn remove(&self) -> io::Result<()> { - match fs::remove_file(&self.path) { - Ok(()) => Ok(()), - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - } - } - } - - pub fn write(&self) -> io::Result<()> { - fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) - } -} diff --git a/standalonex/src/src/utils/helpers/tests.rs b/standalonex/src/src/utils/helpers/tests.rs deleted file mode 100644 index f6fe6f47..00000000 --- a/standalonex/src/src/utils/helpers/tests.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::fs::{self, File, remove_file}; -use std::io::Write; -use std::path::PathBuf; - -use crate::utils::helpers::{ - check_cfg_arg, extract_beta_rev, hex_encode, make, program_out_of_date, set_file_times, - symlink_dir, -}; -use crate::{Config, Flags}; - -#[test] -fn test_make() { - for (host, make_path) in vec![ - ("dragonfly", PathBuf::from("gmake")), - ("netbsd", PathBuf::from("gmake")), - ("freebsd", PathBuf::from("gmake")), - ("openbsd", PathBuf::from("gmake")), - ("linux", PathBuf::from("make")), - // for checking the default - ("_", PathBuf::from("make")), - ] { - assert_eq!(make(host), make_path); - } -} - -#[test] -fn test_beta_rev_parsing() { - // single digit revision - assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); - // multiple digits - assert_eq!(extract_beta_rev("1.99.9-beta.777 (xxxxxx)"), Some("777".to_string())); - // nightly channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9-nightly (xxxxxx)"), None); - // stable channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9 (xxxxxxx)"), None); - // invalid string - assert_eq!(extract_beta_rev("invalid"), None); -} - -#[test] -fn test_string_to_hex_encode() { - let input_string = "Hello, World!"; - let hex_string = hex_encode(input_string); - assert_eq!(hex_string, "48656c6c6f2c20576f726c6421"); -} - -#[test] -fn test_check_cfg_arg() { - assert_eq!(check_cfg_arg("bootstrap", None), "--check-cfg=cfg(bootstrap)"); - assert_eq!( - check_cfg_arg("target_arch", Some(&["s360"])), - "--check-cfg=cfg(target_arch,values(\"s360\"))" - ); - assert_eq!( - check_cfg_arg("target_os", Some(&["nixos", "nix2"])), - "--check-cfg=cfg(target_os,values(\"nixos\",\"nix2\"))" - ); -} - -#[test] -fn test_program_out_of_date() { - let config = - Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); - let tempfile = config.tempdir().join(".tmp-stamp-file"); - File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); - assert!(tempfile.exists()); - - // up-to-date - assert!(!program_out_of_date(&tempfile, "dummy value")); - // out-of-date - assert!(program_out_of_date(&tempfile, "")); - - remove_file(tempfile).unwrap(); -} - -#[test] -fn test_symlink_dir() { - let config = - Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); - let tempdir = config.tempdir().join(".tmp-dir"); - let link_path = config.tempdir().join(".tmp-link"); - - fs::create_dir_all(&tempdir).unwrap(); - symlink_dir(&config, &tempdir, &link_path).unwrap(); - - let link_source = fs::read_link(&link_path).unwrap(); - assert_eq!(link_source, tempdir); - - fs::remove_dir(tempdir).unwrap(); - - #[cfg(windows)] - fs::remove_dir(link_path).unwrap(); - #[cfg(not(windows))] - fs::remove_file(link_path).unwrap(); -} - -#[test] -fn test_set_file_times_sanity_check() { - let config = - Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); - let tempfile = config.tempdir().join(".tmp-file"); - - { - File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); - assert!(tempfile.exists()); - } - - // This might only fail on Windows (if file is default read-only then we try to modify file - // times). - let unix_epoch = std::time::SystemTime::UNIX_EPOCH; - let target_time = fs::FileTimes::new().set_accessed(unix_epoch).set_modified(unix_epoch); - set_file_times(&tempfile, target_time).unwrap(); - - let found_metadata = fs::metadata(tempfile).unwrap(); - assert_eq!(found_metadata.accessed().unwrap(), unix_epoch); - assert_eq!(found_metadata.modified().unwrap(), unix_epoch) -} diff --git a/standalonex/src/src/utils/job.rs b/standalonex/src/src/utils/job.rs deleted file mode 100644 index c5e89245..00000000 --- a/standalonex/src/src/utils/job.rs +++ /dev/null @@ -1,154 +0,0 @@ -#[cfg(windows)] -pub use for_windows::*; - -#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] -pub unsafe fn setup(_build: &mut crate::Build) {} - -#[cfg(all(unix, not(target_os = "haiku")))] -pub unsafe fn setup(build: &mut crate::Build) { - if build.config.low_priority { - libc::setpriority(libc::PRIO_PGRP as _, 0, 10); - } -} - -/// Job management on Windows for bootstrapping -/// -/// Most of the time when you're running a build system (e.g., make) you expect -/// Ctrl-C or abnormal termination to actually terminate the entire tree of -/// process in play, not just the one at the top. This currently works "by -/// default" on Unix platforms because Ctrl-C actually sends a signal to the -/// *process group* rather than the parent process, so everything will get torn -/// down. On Windows, however, this does not happen and Ctrl-C just kills the -/// parent process. -/// -/// To achieve the same semantics on Windows we use Job Objects to ensure that -/// all processes die at the same time. Job objects have a mode of operation -/// where when all handles to the object are closed it causes all child -/// processes associated with the object to be terminated immediately. -/// Conveniently whenever a process in the job object spawns a new process the -/// child will be associated with the job object as well. This means if we add -/// ourselves to the job object we create then everything will get torn down! -/// -/// Unfortunately most of the time the build system is actually called from a -/// python wrapper (which manages things like building the build system) so this -/// all doesn't quite cut it so far. To go the last mile we duplicate the job -/// object handle into our parent process (a python process probably) and then -/// close our own handle. This means that the only handle to the job object -/// resides in the parent python process, so when python dies the whole build -/// system dies (as one would probably expect!). -/// -/// Note that this is a Windows specific module as none of this logic is required on Unix. -#[cfg(windows)] -mod for_windows { - use std::ffi::c_void; - use std::{env, io, mem}; - - use windows::Win32::Foundation::{CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE}; - use windows::Win32::System::Diagnostics::Debug::{ - SEM_NOGPFAULTERRORBOX, SetErrorMode, THREAD_ERROR_MODE, - }; - use windows::Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, - JOB_OBJECT_LIMIT_PRIORITY_CLASS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JobObjectExtendedLimitInformation, SetInformationJobObject, - }; - use windows::Win32::System::Threading::{ - BELOW_NORMAL_PRIORITY_CLASS, GetCurrentProcess, OpenProcess, PROCESS_DUP_HANDLE, - }; - use windows::core::PCWSTR; - - use crate::Build; - - pub unsafe fn setup(build: &mut Build) { - // Enable the Windows Error Reporting dialog which msys disables, - // so we can JIT debug rustc - let mode = SetErrorMode(THREAD_ERROR_MODE::default()); - let mode = THREAD_ERROR_MODE(mode); - SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); - - // Create a new job object for us to use - let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); - - // Indicate that when all handles to the job object are gone that all - // process in the object should be killed. Note that this includes our - // entire process tree by default because we've added ourselves and our - // children will reside in the job by default. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if build.config.low_priority { - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; - } - let r = SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - &info as *const _ as *const c_void, - mem::size_of_val(&info) as u32, - ); - assert!(r.is_ok(), "{}", io::Error::last_os_error()); - - // Assign our process to this job object. Note that if this fails, one very - // likely reason is that we are ourselves already in a job object! This can - // happen on the build bots that we've got for Windows, or if just anyone - // else is instrumenting the build. In this case we just bail out - // immediately and assume that they take care of it. - // - // Also note that nested jobs (why this might fail) are supported in recent - // versions of Windows, but the version of Windows that our bots are running - // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()); - if r.is_err() { - CloseHandle(job).ok(); - return; - } - - // If we've got a parent process (e.g., the python script that called us) - // then move ownership of this job object up to them. That way if the python - // script is killed (e.g., via ctrl-c) then we'll all be torn down. - // - // If we don't have a parent (e.g., this was run directly) then we - // intentionally leak the job object handle. When our process exits - // (normally or abnormally) it will close the handle implicitly, causing all - // processes in the job to be cleaned up. - let pid = match env::var("BOOTSTRAP_PARENT_ID") { - Ok(s) => s, - Err(..) => return, - }; - - let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { - Some(parent) => parent, - _ => { - // If we get a null parent pointer here, it is possible that either - // we have an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // it might be better to improve the experience of the second case - // when users have interrupted the parent process and we haven't finish - // duplicating the handle yet. We just need close the job object if that occurs. - CloseHandle(job).ok(); - return; - } - }; - - let mut parent_handle = HANDLE::default(); - let r = DuplicateHandle( - GetCurrentProcess(), - job, - parent, - &mut parent_handle, - 0, - false, - DUPLICATE_SAME_ACCESS, - ); - - // If this failed, well at least we tried! An example of DuplicateHandle - // failing in the past has been when the wrong python2 package spawned this - // build system (e.g., the `python2` package in MSYS instead of - // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure - // mode" here is that we only clean everything up when the build system - // dies, not when the python parent does, so not too bad. - if r.is_err() { - CloseHandle(job).ok(); - } - } -} diff --git a/standalonex/src/src/utils/metrics.rs b/standalonex/src/src/utils/metrics.rs deleted file mode 100644 index 3b31fa36..00000000 --- a/standalonex/src/src/utils/metrics.rs +++ /dev/null @@ -1,263 +0,0 @@ -//! This module is responsible for collecting metrics profiling information for the current build -//! and dumping it to disk as JSON, to aid investigations on build and CI performance. -//! -//! As this module requires additional dependencies not present during local builds, it's cfg'd -//! away whenever the `build.metrics` config option is not set to `true`. - -use std::cell::RefCell; -use std::fs::File; -use std::io::BufWriter; -use std::time::{Duration, Instant, SystemTime}; - -use build_helper::metrics::{ - JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, - TestOutcome, TestSuite, TestSuiteMetadata, -}; -use sysinfo::{CpuRefreshKind, RefreshKind, System}; - -use crate::Build; -use crate::core::builder::{Builder, Step}; -use crate::utils::helpers::t; - -// Update this number whenever a breaking change is made to the build metrics. -// -// The output format is versioned for two reasons: -// -// - The metadata is intended to be consumed by external tooling, and exposing a format version -// helps the tools determine whether they're compatible with a metrics file. -// -// - If a developer enables build metrics in their local checkout, making a breaking change to the -// metrics format would result in a hard-to-diagnose error message when an existing metrics file -// is not compatible with the new changes. With a format version number, bootstrap can discard -// incompatible metrics files instead of appending metrics to them. -// -// Version changelog: -// -// - v0: initial version -// - v1: replaced JsonNode::Test with JsonNode::TestSuite -// -const CURRENT_FORMAT_VERSION: usize = 1; - -pub(crate) struct BuildMetrics { - state: RefCell, -} - -/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. -impl Clone for BuildMetrics { - fn clone(&self) -> Self { - Self::init() - } -} - -impl BuildMetrics { - pub(crate) fn init() -> Self { - let state = RefCell::new(MetricsState { - finished_steps: Vec::new(), - running_steps: Vec::new(), - - system_info: System::new_with_specifics( - RefreshKind::new().with_cpu(CpuRefreshKind::everything()), - ), - timer_start: None, - invocation_timer_start: Instant::now(), - invocation_start: SystemTime::now(), - }); - - BuildMetrics { state } - } - - pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - // Consider all the stats gathered so far as the parent's. - if !state.running_steps.is_empty() { - self.collect_stats(&mut *state); - } - - state.system_info.refresh_cpu_usage(); - state.timer_start = Some(Instant::now()); - - state.running_steps.push(StepMetrics { - type_: std::any::type_name::().into(), - debug_repr: format!("{step:?}"), - - cpu_usage_time_sec: 0.0, - duration_excluding_children_sec: Duration::ZERO, - - children: Vec::new(), - test_suites: Vec::new(), - }); - } - - pub(crate) fn exit_step(&self, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - self.collect_stats(&mut *state); - - let step = state.running_steps.pop().unwrap(); - if state.running_steps.is_empty() { - state.finished_steps.push(step); - state.timer_start = None; - } else { - state.running_steps.last_mut().unwrap().children.push(step); - - // Start collecting again for the parent step. - state.system_info.refresh_cpu_usage(); - state.timer_start = Some(Instant::now()); - } - } - - pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - step.test_suites.push(TestSuite { metadata, tests: Vec::new() }); - } - - pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - - if let Some(test_suite) = step.test_suites.last_mut() { - test_suite.tests.push(Test { name: name.to_string(), outcome }); - } else { - panic!("metrics.record_test() called without calling metrics.begin_test_suite() first"); - } - } - - fn collect_stats(&self, state: &mut MetricsState) { - let step = state.running_steps.last_mut().unwrap(); - - let elapsed = state.timer_start.unwrap().elapsed(); - step.duration_excluding_children_sec += elapsed; - - state.system_info.refresh_cpu_usage(); - let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::(); - step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); - } - - pub(crate) fn persist(&self, build: &Build) { - let mut state = self.state.borrow_mut(); - assert!(state.running_steps.is_empty(), "steps are still executing"); - - let dest = build.out.join("metrics.json"); - - let mut system = - System::new_with_specifics(RefreshKind::new().with_cpu(CpuRefreshKind::everything())); - system.refresh_cpu_usage(); - system.refresh_memory(); - - let system_stats = JsonInvocationSystemStats { - cpu_threads_count: system.cpus().len(), - cpu_model: system.cpus()[0].brand().into(), - - memory_total_bytes: system.total_memory(), - }; - let steps = std::mem::take(&mut state.finished_steps); - - // Some of our CI builds consist of multiple independent CI invocations. Ensure all the - // previous invocations are still present in the resulting file. - let mut invocations = match std::fs::read(&dest) { - Ok(contents) => { - // We first parse just the format_version field to have the check succeed even if - // the rest of the contents are not valid anymore. - let version: OnlyFormatVersion = t!(serde_json::from_slice(&contents)); - if version.format_version == CURRENT_FORMAT_VERSION { - t!(serde_json::from_slice::(&contents)).invocations - } else { - println!( - "WARNING: overriding existing build/metrics.json, as it's not \ - compatible with build metrics format version {CURRENT_FORMAT_VERSION}." - ); - Vec::new() - } - } - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - panic!("failed to open existing metrics file at {}: {err}", dest.display()); - } - Vec::new() - } - }; - invocations.push(JsonInvocation { - start_time: state - .invocation_start - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(), - duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), - children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), - }); - - let json = JsonRoot { format_version: CURRENT_FORMAT_VERSION, system_stats, invocations }; - - t!(std::fs::create_dir_all(dest.parent().unwrap())); - let mut file = BufWriter::new(t!(File::create(&dest))); - t!(serde_json::to_writer(&mut file, &json)); - } - - fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { - let mut children = Vec::new(); - children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); - children.extend(step.test_suites.into_iter().map(JsonNode::TestSuite)); - - JsonNode::RustbuildStep { - type_: step.type_, - debug_repr: step.debug_repr, - - duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), - system_stats: JsonStepSystemStats { - cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 - / step.duration_excluding_children_sec.as_secs_f64(), - }, - - children, - } - } -} - -struct MetricsState { - finished_steps: Vec, - running_steps: Vec, - - system_info: System, - timer_start: Option, - invocation_timer_start: Instant, - invocation_start: SystemTime, -} - -struct StepMetrics { - type_: String, - debug_repr: String, - - cpu_usage_time_sec: f64, - duration_excluding_children_sec: Duration, - - children: Vec, - test_suites: Vec, -} - -#[derive(serde_derive::Deserialize)] -struct OnlyFormatVersion { - #[serde(default)] // For version 0 the field was not present. - format_version: usize, -} diff --git a/standalonex/src/src/utils/mod.rs b/standalonex/src/src/utils/mod.rs deleted file mode 100644 index 53b41f15..00000000 --- a/standalonex/src/src/utils/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! This module contains integral components of the build and configuration process, providing -//! support for a wide range of tasks and operations such as caching, tarballs, release -//! channels, job management, etc. - -pub(crate) mod cache; -pub(crate) mod cc_detect; -pub(crate) mod change_tracker; -pub(crate) mod channel; -pub(crate) mod exec; -pub(crate) mod helpers; -pub(crate) mod job; -#[cfg(feature = "build-metrics")] -pub(crate) mod metrics; -pub(crate) mod render_tests; -pub(crate) mod shared_helpers; -pub(crate) mod tarball; diff --git a/standalonex/src/src/utils/render_tests.rs b/standalonex/src/src/utils/render_tests.rs deleted file mode 100644 index eb2c8254..00000000 --- a/standalonex/src/src/utils/render_tests.rs +++ /dev/null @@ -1,434 +0,0 @@ -//! This module renders the JSON output of libtest into a human-readable form, trying to be as -//! similar to libtest's native output as possible. -//! -//! This is needed because we need to use libtest in JSON mode to extract granular information -//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo -//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had -//! to reimplement all the rendering logic in this module because of that. - -use std::io::{BufRead, BufReader, Read, Write}; -use std::process::{ChildStdout, Stdio}; -use std::time::Duration; - -use termcolor::{Color, ColorSpec, WriteColor}; - -use crate::core::builder::Builder; -use crate::utils::exec::BootstrapCommand; - -const TERSE_TESTS_PER_LINE: usize = 88; - -pub(crate) fn add_flags_and_try_run_tests( - builder: &Builder<'_>, - cmd: &mut BootstrapCommand, -) -> bool { - if !cmd.get_args().any(|arg| arg == "--") { - cmd.arg("--"); - } - cmd.args(["-Z", "unstable-options", "--format", "json"]); - - try_run_tests(builder, cmd, false) -} - -pub(crate) fn try_run_tests( - builder: &Builder<'_>, - cmd: &mut BootstrapCommand, - stream: bool, -) -> bool { - if builder.config.dry_run() { - cmd.mark_as_executed(); - return true; - } - - if !run_tests(builder, cmd, stream) { - if builder.fail_fast { - crate::exit!(1); - } else { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - false - } - } else { - true - } -} - -fn run_tests(builder: &Builder<'_>, cmd: &mut BootstrapCommand, stream: bool) -> bool { - let cmd = cmd.as_command_mut(); - cmd.stdout(Stdio::piped()); - - builder.verbose(|| println!("running: {cmd:?}")); - - let mut process = cmd.spawn().unwrap(); - - // This runs until the stdout of the child is closed, which means the child exited. We don't - // run this on another thread since the builder is not Sync. - let renderer = Renderer::new(process.stdout.take().unwrap(), builder); - if stream { - renderer.stream_all(); - } else { - renderer.render_all(); - } - - let result = process.wait_with_output().unwrap(); - if !result.status.success() && builder.is_verbose() { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {}", - result.status - ); - } - - result.status.success() -} - -struct Renderer<'a> { - stdout: BufReader, - failures: Vec, - benches: Vec, - builder: &'a Builder<'a>, - tests_count: Option, - executed_tests: usize, - /// Number of tests that were skipped due to already being up-to-date - /// (i.e. no relevant changes occurred since they last ran). - up_to_date_tests: usize, - terse_tests_in_line: usize, -} - -impl<'a> Renderer<'a> { - fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { - Self { - stdout: BufReader::new(stdout), - benches: Vec::new(), - failures: Vec::new(), - builder, - tests_count: None, - executed_tests: 0, - up_to_date_tests: 0, - terse_tests_in_line: 0, - } - } - - fn render_all(mut self) { - let mut line = Vec::new(); - loop { - line.clear(); - match self.stdout.read_until(b'\n', &mut line) { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - if line.is_empty() { - break; - } - - match serde_json::from_slice(&line) { - Ok(parsed) => self.render_message(parsed), - Err(_err) => { - // Handle non-JSON output, for example when --nocapture is passed. - let mut stdout = std::io::stdout(); - stdout.write_all(&line).unwrap(); - let _ = stdout.flush(); - } - } - } - - if self.up_to_date_tests > 0 { - let n = self.up_to_date_tests; - let s = if n > 1 { "s" } else { "" }; - println!("help: ignored {n} up-to-date test{s}; use `--force-rerun` to prevent this\n"); - } - } - - /// Renders the stdout characters one by one - fn stream_all(mut self) { - let mut buffer = [0; 1]; - loop { - match self.stdout.read(&mut buffer) { - Ok(0) => break, - Ok(_) => { - let mut stdout = std::io::stdout(); - stdout.write_all(&buffer).unwrap(); - let _ = stdout.flush(); - } - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - } - } - - fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { - self.executed_tests += 1; - - // Keep this in sync with the "up-to-date" ignore message inserted by compiletest. - if let Outcome::Ignored { reason: Some("up-to-date") } = outcome { - self.up_to_date_tests += 1; - } - - #[cfg(feature = "build-metrics")] - self.builder.metrics.record_test( - &test.name, - match outcome { - Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, - Outcome::Failed => build_helper::metrics::TestOutcome::Failed, - Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { - ignore_reason: reason.map(|s| s.to_string()), - }, - }, - self.builder, - ); - - if self.builder.config.verbose_tests { - self.render_test_outcome_verbose(outcome, test); - } else { - self.render_test_outcome_terse(outcome, test); - } - } - - fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { - print!("test {} ... ", test.name); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - if let Some(exec_time) = test.exec_time { - print!(" ({exec_time:.2?})"); - } - println!(); - } - - fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { - if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { - if let Some(total) = self.tests_count { - let total = total.to_string(); - let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); - print!(" {executed}/{total}"); - } - println!(); - self.terse_tests_in_line = 0; - } - - self.terse_tests_in_line += 1; - self.builder.colored_stdout(|stdout| outcome.write_short(stdout, &test.name)).unwrap(); - let _ = std::io::stdout().flush(); - } - - fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { - // The terse output doesn't end with a newline, so we need to add it ourselves. - if !self.builder.config.verbose_tests { - println!(); - } - - if !self.failures.is_empty() { - println!("\nfailures:\n"); - for failure in &self.failures { - if failure.stdout.is_some() || failure.message.is_some() { - println!("---- {} stdout ----", failure.name); - if let Some(stdout) = &failure.stdout { - println!("{stdout}"); - } - if let Some(message) = &failure.message { - println!("NOTE: {message}"); - } - } - } - - println!("\nfailures:"); - for failure in &self.failures { - println!(" {}", failure.name); - } - } - - if !self.benches.is_empty() { - println!("\nbenchmarks:"); - - let mut rows = Vec::new(); - for bench in &self.benches { - rows.push(( - &bench.name, - format!("{:.2?}/iter", bench.median), - format!("+/- {:.2?}", bench.deviation), - )); - } - - let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); - let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); - let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); - for row in &rows { - println!(" {:max_1$} {:>max_2$}", row.0, row.1, row.2); - } - } - - print!("\ntest result: "); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - println!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out{time}\n", - suite.passed, - suite.failed, - suite.ignored, - suite.measured, - suite.filtered_out, - time = match suite.exec_time { - Some(t) => format!("; finished in {:.2?}", Duration::from_secs_f64(t)), - None => String::new(), - } - ); - } - - fn render_message(&mut self, message: Message) { - match message { - Message::Suite(SuiteMessage::Started { test_count }) => { - println!("\nrunning {test_count} tests"); - self.executed_tests = 0; - self.terse_tests_in_line = 0; - self.tests_count = Some(test_count); - } - Message::Suite(SuiteMessage::Ok(outcome)) => { - self.render_suite_outcome(Outcome::Ok, &outcome); - } - Message::Suite(SuiteMessage::Failed(outcome)) => { - self.render_suite_outcome(Outcome::Failed, &outcome); - } - Message::Bench(outcome) => { - // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest - // outputs, mostly because libtest's formatting is broken in terse mode, which is - // the default used by our monorepo. We use a different formatting instead: - // successful benchmarks are just showed as "benchmarked"/"b", and the details are - // outputted at the bottom like failures. - let fake_test_outcome = TestOutcome { - name: outcome.name.clone(), - exec_time: None, - stdout: None, - message: None, - }; - self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); - self.benches.push(outcome); - } - Message::Test(TestMessage::Ok(outcome)) => { - self.render_test_outcome(Outcome::Ok, &outcome); - } - Message::Test(TestMessage::Ignored(outcome)) => { - self.render_test_outcome( - Outcome::Ignored { reason: outcome.message.as_deref() }, - &outcome, - ); - } - Message::Test(TestMessage::Failed(outcome)) => { - self.render_test_outcome(Outcome::Failed, &outcome); - self.failures.push(outcome); - } - Message::Test(TestMessage::Timeout { name }) => { - println!("test {name} has been running for a long time"); - } - Message::Test(TestMessage::Started) => {} // Not useful - } - } -} - -enum Outcome<'a> { - Ok, - BenchOk, - Failed, - Ignored { reason: Option<&'a str> }, -} - -impl Outcome<'_> { - fn write_short(&self, writer: &mut dyn WriteColor, name: &str) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, ".")?; - } - Outcome::BenchOk => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "b")?; - } - Outcome::Failed => { - // Put failed tests on their own line and include the test name, so that it's faster - // to see which test failed without having to wait for them all to run. - writeln!(writer)?; - writer.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?; - writeln!(writer, "{name} ... F")?; - } - Outcome::Ignored { .. } => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "i")?; - } - } - writer.reset() - } - - fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, "ok")?; - } - Outcome::BenchOk => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "benchmarked")?; - } - Outcome::Failed => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "FAILED")?; - } - Outcome::Ignored { reason } => { - writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "ignored")?; - if let Some(reason) = reason { - write!(writer, ", {reason}")?; - } - } - } - writer.reset() - } -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum Message { - Suite(SuiteMessage), - Test(TestMessage), - Bench(BenchOutcome), -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum SuiteMessage { - Ok(SuiteOutcome), - Failed(SuiteOutcome), - Started { test_count: usize }, -} - -#[derive(serde_derive::Deserialize)] -struct SuiteOutcome { - passed: usize, - failed: usize, - ignored: usize, - measured: usize, - filtered_out: usize, - /// The time it took to execute this test suite, or `None` if time measurement was not possible - /// (e.g. due to running on wasm). - exec_time: Option, -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum TestMessage { - Ok(TestOutcome), - Failed(TestOutcome), - Ignored(TestOutcome), - Timeout { name: String }, - Started, -} - -#[derive(serde_derive::Deserialize)] -struct BenchOutcome { - name: String, - median: f64, - deviation: f64, -} - -#[derive(serde_derive::Deserialize)] -struct TestOutcome { - name: String, - exec_time: Option, - stdout: Option, - message: Option, -} diff --git a/standalonex/src/src/utils/shared_helpers.rs b/standalonex/src/src/utils/shared_helpers.rs deleted file mode 100644 index 6d3c276c..00000000 --- a/standalonex/src/src/utils/shared_helpers.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! This module serves two purposes: -//! 1. It is part of the `utils` module and used in other parts of bootstrap. -//! 2. It is embedded inside bootstrap shims to avoid a dependency on the bootstrap library. -//! Therefore, this module should never use any other bootstrap module. This reduces binary -//! size and improves compilation time by minimizing linking time. - -#![allow(dead_code)] - -use std::env; -use std::ffi::OsString; -use std::fs::OpenOptions; -use std::io::Write; -use std::process::Command; -use std::str::FromStr; - -#[cfg(test)] -mod tests; - -/// Returns the environment variable which the dynamic library lookup path -/// resides in for this platform. -pub fn dylib_path_var() -> &'static str { - if cfg!(target_os = "windows") { - "PATH" - } else if cfg!(target_vendor = "apple") { - "DYLD_LIBRARY_PATH" - } else if cfg!(target_os = "haiku") { - "LIBRARY_PATH" - } else if cfg!(target_os = "aix") { - "LIBPATH" - } else { - "LD_LIBRARY_PATH" - } -} - -/// Parses the `dylib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -pub fn dylib_path() -> Vec { - let var = match std::env::var_os(dylib_path_var()) { - Some(v) => v, - None => return vec![], - }; - std::env::split_paths(&var).collect() -} - -/// Given an executable called `name`, return the filename for the -/// executable for a particular target. -pub fn exe(name: &str, target: &str) -> String { - if target.contains("windows") { - format!("{name}.exe") - } else if target.contains("uefi") { - format!("{name}.efi") - } else if target.contains("wasm") { - format!("{name}.wasm") - } else { - name.to_string() - } -} - -/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. -/// If it was not defined, returns 0 by default. -/// -/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. -pub fn parse_rustc_verbose() -> usize { - match env::var("RUSTC_VERBOSE") { - Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), - Err(_) => 0, - } -} - -/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. -/// -/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. -pub fn parse_rustc_stage() -> String { - env::var("RUSTC_STAGE").unwrap_or_else(|_| { - // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. - eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set"); - eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap"); - std::process::exit(101); - }) -} - -/// Writes the command invocation to a file if `DUMP_BOOTSTRAP_SHIMS` is set during bootstrap. -/// -/// Before writing it, replaces user-specific values to create generic dumps for cross-environment -/// comparisons. -pub fn maybe_dump(dump_name: String, cmd: &Command) { - if let Ok(dump_dir) = env::var("DUMP_BOOTSTRAP_SHIMS") { - let dump_file = format!("{dump_dir}/{dump_name}"); - - let mut file = OpenOptions::new().create(true).append(true).open(dump_file).unwrap(); - - let cmd_dump = format!("{:?}\n", cmd); - let cmd_dump = cmd_dump.replace(&env::var("BUILD_OUT").unwrap(), "${BUILD_OUT}"); - let cmd_dump = cmd_dump.replace(&env::var("CARGO_HOME").unwrap(), "${CARGO_HOME}"); - - file.write_all(cmd_dump.as_bytes()).expect("Unable to write file"); - } -} - -/// Finds `key` and returns its value from the given list of arguments `args`. -pub fn parse_value_from_args<'a>(args: &'a [OsString], key: &str) -> Option<&'a str> { - let mut args = args.iter(); - while let Some(arg) = args.next() { - let arg = arg.to_str().unwrap(); - - if let Some(value) = arg.strip_prefix(&format!("{key}=")) { - return Some(value); - } else if arg == key { - return args.next().map(|v| v.to_str().unwrap()); - } - } - - None -} diff --git a/standalonex/src/src/utils/shared_helpers/tests.rs b/standalonex/src/src/utils/shared_helpers/tests.rs deleted file mode 100644 index da792427..00000000 --- a/standalonex/src/src/utils/shared_helpers/tests.rs +++ /dev/null @@ -1,28 +0,0 @@ -use super::parse_value_from_args; - -#[test] -fn test_parse_value_from_args() { - let args = vec![ - "--stage".into(), - "1".into(), - "--version".into(), - "2".into(), - "--target".into(), - "x86_64-unknown-linux".into(), - ]; - - assert_eq!(parse_value_from_args(args.as_slice(), "--stage").unwrap(), "1"); - assert_eq!(parse_value_from_args(args.as_slice(), "--version").unwrap(), "2"); - assert_eq!(parse_value_from_args(args.as_slice(), "--target").unwrap(), "x86_64-unknown-linux"); - assert!(parse_value_from_args(args.as_slice(), "random-key").is_none()); - - let args = vec![ - "app-name".into(), - "--key".into(), - "value".into(), - "random-value".into(), - "--sysroot=/x/y/z".into(), - ]; - assert_eq!(parse_value_from_args(args.as_slice(), "--key").unwrap(), "value"); - assert_eq!(parse_value_from_args(args.as_slice(), "--sysroot").unwrap(), "/x/y/z"); -} diff --git a/standalonex/src/src/utils/tarball.rs b/standalonex/src/src/utils/tarball.rs deleted file mode 100644 index 3c6c7a7f..00000000 --- a/standalonex/src/src/utils/tarball.rs +++ /dev/null @@ -1,439 +0,0 @@ -//! Facilitates the management and generation of tarballs. -//! -//! Tarballs efficiently hold Rust compiler build artifacts and -//! capture a snapshot of each boostrap stage. -//! In uplifting, a tarball from Stage N captures essential components -//! to assemble Stage N + 1 compiler. - -use std::path::{Path, PathBuf}; - -use crate::core::build_steps::dist::distdir; -use crate::core::builder::{Builder, Kind}; -use crate::core::config::BUILDER_CONFIG_FILENAME; -use crate::utils::exec::BootstrapCommand; -use crate::utils::helpers::{move_file, t}; -use crate::utils::{channel, helpers}; - -#[derive(Copy, Clone)] -pub(crate) enum OverlayKind { - Rust, - Llvm, - Cargo, - Clippy, - Miri, - Rustfmt, - Rls, - RustAnalyzer, - RustcCodegenCranelift, - LlvmBitcodeLinker, -} - -impl OverlayKind { - fn legal_and_readme(&self) -> &[&str] { - match self { - OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], - OverlayKind::Llvm => { - &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] - } - OverlayKind::Cargo => &[ - "src/tools/cargo/README.md", - "src/tools/cargo/LICENSE-MIT", - "src/tools/cargo/LICENSE-APACHE", - "src/tools/cargo/LICENSE-THIRD-PARTY", - ], - OverlayKind::Clippy => &[ - "src/tools/clippy/README.md", - "src/tools/clippy/LICENSE-APACHE", - "src/tools/clippy/LICENSE-MIT", - ], - OverlayKind::Miri => &[ - "src/tools/miri/README.md", - "src/tools/miri/LICENSE-APACHE", - "src/tools/miri/LICENSE-MIT", - ], - OverlayKind::Rustfmt => &[ - "src/tools/rustfmt/README.md", - "src/tools/rustfmt/LICENSE-APACHE", - "src/tools/rustfmt/LICENSE-MIT", - ], - OverlayKind::Rls => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], - OverlayKind::RustAnalyzer => &[ - "src/tools/rust-analyzer/README.md", - "src/tools/rust-analyzer/LICENSE-APACHE", - "src/tools/rust-analyzer/LICENSE-MIT", - ], - OverlayKind::RustcCodegenCranelift => &[ - "compiler/rustc_codegen_cranelift/Readme.md", - "compiler/rustc_codegen_cranelift/LICENSE-APACHE", - "compiler/rustc_codegen_cranelift/LICENSE-MIT", - ], - OverlayKind::LlvmBitcodeLinker => &[ - "COPYRIGHT", - "LICENSE-APACHE", - "LICENSE-MIT", - "src/tools/llvm-bitcode-linker/README.md", - ], - } - } - - fn version(&self, builder: &Builder<'_>) -> String { - match self { - OverlayKind::Rust => builder.rust_version(), - OverlayKind::Llvm => builder.rust_version(), - OverlayKind::Cargo => { - builder.cargo_info.version(builder, &builder.release_num("cargo")) - } - OverlayKind::Clippy => { - builder.clippy_info.version(builder, &builder.release_num("clippy")) - } - OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), - OverlayKind::Rustfmt => { - builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) - } - OverlayKind::Rls => builder.release(&builder.release_num("rls")), - OverlayKind::RustAnalyzer => builder - .rust_analyzer_info - .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), - OverlayKind::RustcCodegenCranelift => builder.rust_version(), - OverlayKind::LlvmBitcodeLinker => builder.rust_version(), - } - } -} - -pub(crate) struct Tarball<'a> { - builder: &'a Builder<'a>, - - pkgname: String, - component: String, - target: Option, - product_name: String, - overlay: OverlayKind, - - temp_dir: PathBuf, - image_dir: PathBuf, - overlay_dir: PathBuf, - bulk_dirs: Vec, - - include_target_in_component_name: bool, - is_preview: bool, - permit_symlinks: bool, -} - -impl<'a> Tarball<'a> { - pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { - Self::new_inner(builder, component, Some(target.into())) - } - - pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { - Self::new_inner(builder, component, None) - } - - fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { - let pkgname = crate::core::build_steps::dist::pkgname(builder, component); - - let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); - if let Some(target) = &target { - temp_dir = temp_dir.join(target); - } - let _ = std::fs::remove_dir_all(&temp_dir); - - let image_dir = temp_dir.join("image"); - let overlay_dir = temp_dir.join("overlay"); - - Self { - builder, - - pkgname, - component: component.into(), - target, - product_name: "Rust".into(), - overlay: OverlayKind::Rust, - - temp_dir, - image_dir, - overlay_dir, - bulk_dirs: Vec::new(), - - include_target_in_component_name: false, - is_preview: false, - permit_symlinks: false, - } - } - - pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { - self.overlay = overlay; - } - - pub(crate) fn set_product_name(&mut self, name: &str) { - self.product_name = name.into(); - } - - pub(crate) fn include_target_in_component_name(&mut self, include: bool) { - self.include_target_in_component_name = include; - } - - pub(crate) fn is_preview(&mut self, is: bool) { - self.is_preview = is; - } - - pub(crate) fn permit_symlinks(&mut self, flag: bool) { - self.permit_symlinks = flag; - } - - pub(crate) fn image_dir(&self) -> &Path { - t!(std::fs::create_dir_all(&self.image_dir)); - &self.image_dir - } - - pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { - // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply - // uses the base directory as the destination directory. - let destdir = if destdir.as_ref() == Path::new(".") { - self.image_dir.clone() - } else { - self.image_dir.join(destdir.as_ref()) - }; - - t!(std::fs::create_dir_all(&destdir)); - self.builder.install(src.as_ref(), &destdir, perms); - } - - pub(crate) fn add_renamed_file( - &self, - src: impl AsRef, - destdir: impl AsRef, - new_name: &str, - ) { - let destdir = self.image_dir.join(destdir.as_ref()); - t!(std::fs::create_dir_all(&destdir)); - self.builder.copy_link(src.as_ref(), &destdir.join(new_name)); - } - - pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { - for file in self.overlay.legal_and_readme() { - self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); - } - } - - pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { - let dest = self.image_dir.join(dest.as_ref()); - - t!(std::fs::create_dir_all(&dest)); - self.builder.cp_link_r(src.as_ref(), &dest); - } - - pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { - self.bulk_dirs.push(dest.as_ref().to_path_buf()); - self.add_dir(src, dest); - } - - pub(crate) fn generate(self) -> GeneratedTarball { - let mut component_name = self.component.clone(); - if self.is_preview { - component_name.push_str("-preview"); - } - if self.include_target_in_component_name { - component_name.push('-'); - component_name.push_str( - self.target - .as_ref() - .expect("include_target_in_component_name used in a targetless tarball"), - ); - } - - self.run(|this, cmd| { - cmd.arg("generate") - .arg("--image-dir") - .arg(&this.image_dir) - .arg(format!("--component-name={component_name}")); - - if let Some((dir, dirs)) = this.bulk_dirs.split_first() { - let mut arg = dir.as_os_str().to_os_string(); - for dir in dirs { - arg.push(","); - arg.push(dir); - } - cmd.arg("--bulk-dirs").arg(&arg); - } - - this.non_bare_args(cmd); - }) - } - - pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { - let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(&tarball.path); - } - - self.run(|this, cmd| { - cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); - this.non_bare_args(cmd); - }) - } - - pub(crate) fn bare(self) -> GeneratedTarball { - // Bare tarballs should have the top level directory match the package - // name, not "image". We rename the image directory just before passing - // into rust-installer. - let dest = self.temp_dir.join(self.package_name()); - t!(move_file(&self.image_dir, &dest)); - - self.run(|this, cmd| { - let distdir = distdir(this.builder); - t!(std::fs::create_dir_all(&distdir)); - cmd.arg("tarball") - .arg("--input") - .arg(&dest) - .arg("--output") - .arg(distdir.join(this.package_name())); - }) - } - - fn package_name(&self) -> String { - if let Some(target) = &self.target { - format!("{}-{}", self.pkgname, target) - } else { - self.pkgname.clone() - } - } - - fn non_bare_args(&self, cmd: &mut BootstrapCommand) { - cmd.arg("--rel-manifest-dir=rustlib") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--product-name={}", self.product_name)) - .arg(format!("--success-message={} installed.", self.component)) - .arg(format!("--package-name={}", self.package_name())) - .arg("--non-installed-overlay") - .arg(&self.overlay_dir) - .arg("--output-dir") - .arg(distdir(self.builder)); - } - - fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball { - t!(std::fs::create_dir_all(&self.overlay_dir)); - self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); - if let Some(info) = self.builder.rust_info().info() { - channel::write_commit_hash_file(&self.overlay_dir, &info.sha); - channel::write_commit_info_file(&self.overlay_dir, info); - } - - // Add config file if present. - if let Some(config) = &self.builder.config.config { - self.add_renamed_file(config, &self.overlay_dir, BUILDER_CONFIG_FILENAME); - } - - for file in self.overlay.legal_and_readme() { - self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); - } - - let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller); - - let package_name = self.package_name(); - self.builder.info(&format!("Dist {package_name}")); - let _time = crate::utils::helpers::timeit(self.builder); - - build_cli(&self, &mut cmd); - cmd.arg("--work-dir").arg(&self.temp_dir); - if let Some(formats) = &self.builder.config.dist_compression_formats { - assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); - cmd.arg("--compression-formats").arg(formats.join(",")); - } - - // For `x install` tarball files aren't needed, so we can speed up the process by not producing them. - let compression_profile = if self.builder.kind == Kind::Install { - self.builder.verbose(|| { - println!("Forcing dist.compression-profile = 'no-op' for `x install`.") - }); - // "no-op" indicates that the rust-installer won't produce compressed tarball sources. - "no-op" - } else { - assert!( - self.builder.config.dist_compression_profile != "no-op", - "dist.compression-profile = 'no-op' can only be used for `x install`" - ); - - &self.builder.config.dist_compression_profile - }; - - cmd.args(["--compression-profile", compression_profile]); - - // We want to use a pinned modification time for files in the archive - // to achieve better reproducibility. However, using the same mtime for all - // releases is not ideal, because it can break e.g. Cargo mtime checking - // (https://github.com/rust-lang/rust/issues/125578). - // Therefore, we set mtime to the date of the latest commit (if we're managed - // by git). In this way, the archive will still be always the same for a given commit - // (achieving reproducibility), but it will also change between different commits and - // Rust versions, so that it won't break mtime-based caches. - // - // Note that this only overrides the mtime of files, not directories, due to the - // limitations of the tarballer tool. Directories will have their mtime set to 2006. - - // Get the UTC timestamp of the last git commit, if we're under git. - // We need to use UTC, so that anyone who tries to rebuild from the same commit - // gets the same timestamp. - if self.builder.rust_info().is_managed_git_subrepository() { - // %ct means committer date - let timestamp = helpers::git(Some(&self.builder.src)) - .arg("log") - .arg("-1") - .arg("--format=%ct") - .run_capture_stdout(self.builder) - .stdout(); - cmd.args(["--override-file-mtime", timestamp.trim()]); - } - - cmd.run(self.builder); - - // Ensure there are no symbolic links in the tarball. In particular, - // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. - let decompressed_output = self.temp_dir.join(&package_name); - if !self.builder.config.dry_run() && !self.permit_symlinks { - for entry in walkdir::WalkDir::new(&decompressed_output) { - let entry = t!(entry); - if entry.path_is_symlink() { - panic!("generated a symlink in a tarball: {}", entry.path().display()); - } - } - } - - // Use either the first compression format defined, or "gz" as the default. - let ext = self - .builder - .config - .dist_compression_formats - .as_ref() - .and_then(|formats| formats.first()) - .map(|s| s.as_str()) - .unwrap_or("gz"); - - GeneratedTarball { - path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")), - decompressed_output, - work: self.temp_dir, - } - } -} - -#[derive(Debug, Clone)] -pub struct GeneratedTarball { - path: PathBuf, - decompressed_output: PathBuf, - work: PathBuf, -} - -impl GeneratedTarball { - pub(crate) fn tarball(&self) -> &Path { - &self.path - } - - pub(crate) fn decompressed_output(&self) -> &Path { - &self.decompressed_output - } - - pub(crate) fn work_dir(&self) -> &Path { - &self.work - } -} diff --git a/standalonex/src/stage0_parser_crate/Cargo.toml b/standalonex/src/stage0_parser_crate/Cargo.toml new file mode 100644 index 00000000..ae4f0bda --- /dev/null +++ b/standalonex/src/stage0_parser_crate/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "stage0_parser_crate" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = "1" +serde_derive = "1" diff --git a/standalonex/src/stage0_parser_crate/src/lib.rs b/standalonex/src/stage0_parser_crate/src/lib.rs new file mode 100644 index 00000000..2e985572 --- /dev/null +++ b/standalonex/src/stage0_parser_crate/src/lib.rs @@ -0,0 +1,78 @@ +use std::collections::BTreeMap; +use std::path::Path; + +#[derive(Default, Clone)] +pub struct Stage0 { + pub compiler: VersionMetadata, + pub rustfmt: Option, + pub config: Stage0Config, + pub checksums_sha256: BTreeMap, +} + +#[derive(Default, Clone)] +pub struct VersionMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub git_repository: String, + pub nightly_branch: String, +} + +pub fn parse_stage0_file(path: &Path) -> Stage0 { + let stage0_content = std::fs::read_to_string(path) + .expect(&format!("Failed to read stage0 file: {}", path.display())); + + let mut stage0 = Stage0::default(); + for line in stage0_content.lines() { + let line = line.trim(); + + if line.is_empty() { + continue; + } + + // Ignore comments + if line.starts_with('#') { + continue; + } + + let (key, value) = line.split_once('=').unwrap(); + + match key { + "dist_server" => stage0.config.dist_server = value.to_owned(), + "artifacts_server" => stage0.config.artifacts_server = value.to_owned(), + "artifacts_with_llvm_assertions_server" => { + stage0.config.artifacts_with_llvm_assertions_server = value.to_owned() + } + "git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(), + "git_repository" => stage0.config.git_repository = value.to_owned(), + "nightly_branch" => stage0.config.nightly_branch = value.to_owned(), + + "compiler_date" => stage0.compiler.date = value.to_owned(), + "compiler_version" => stage0.compiler.version = value.to_owned(), + + "rustfmt_date" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned(); + } + "rustfmt_version" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).version = value.to_owned(); + } + + dist if dist.starts_with("dist") => { + stage0.checksums_sha256.insert(key.to_owned(), value.to_owned()); + } + + unsupported => { + println!("'{unsupported}' field is not supported."); + } + } + } + + stage0 +} diff --git a/standalonex/src/version b/standalonex/src/version new file mode 100644 index 00000000..d944efab --- /dev/null +++ b/standalonex/src/version @@ -0,0 +1 @@ +1.84.1 \ No newline at end of file diff --git a/standalonex/task.md b/standalonex/task.md deleted file mode 100644 index 3340fbb8..00000000 --- a/standalonex/task.md +++ /dev/null @@ -1,50 +0,0 @@ -# Outstanding Work for Nixification of Rust Bootstrap - -This document outlines the remaining tasks to achieve the goal of having `x.py` emit Nix expressions for Rust compiler calls, effectively turning `x.py` into a Nix package generator. - -## Goal - -To generate a series of Nix flakes, each representing a single compiler invocation within the Rust bootstrap process, allowing the entire Rust build to be driven purely by Nix. The key is to *not* execute `rustc` or `cargo` directly from `x.py`, but instead to have `x.py` emit a Nix expression that represents that compiler call. - -## Outstanding Tasks - -1. **Modify `bootstrap.py` to Emit Nix Expressions (High Priority)** - * **Intercept `run` Function:** The `run` function in `bootstrap.py` is responsible for executing external commands, including `rustc` and `cargo`. This function needs to be modified. - * **Replace Execution with Nix Expression Generation:** Instead of executing `subprocess.Popen(args, **kwargs)`, the `run` function should: - * Construct a JSON object containing the details of the intended command invocation: - * `command`: The executable (e.g., `cargo`, `rustc`). - * `args`: A list of arguments passed to the command. - * `env`: A dictionary of environment variables set for the command (consider filtering sensitive or irrelevant variables). - * `cwd`: The current working directory from which the command would have been executed. - * `type`: A string identifier, e.g., "rust_compiler_invocation", to categorize this emitted data. - * Print this JSON object to `stdout`. - * Exit successfully (`sys.exit(0)`) to prevent the actual execution of the Rust compiler and signal that the Nix expression has been emitted. - * **Define Nix Expression Schema:** Establish a clear, structured schema for the emitted JSON objects. This schema will be crucial for the Nix wrapper to correctly interpret the data. - * **Consider LLM Processing and Symbolic Representation:** The JSON schema should be designed to facilitate processing by Large Language Models (LLMs) and allow for symbolic representation (e.g., using emojis or primes as identifiers) for easier manipulation and reflection within a Nix REPL environment. - -2. **Modify `test-rust2/standalonex/flake.nix` `buildPhase` to Process Nix Expressions (High Priority)** - * **Run `python3 x.py build`:** Execute `x.py` (which will now emit JSON Nix expressions to `stdout`). - * **Capture Nix Expression Output:** Capture the `stdout` of `x.py` containing the emitted JSON objects. - * **Process and Generate Virtual Flakes:** Parse the captured JSON objects and dynamically generate new Nix flakes (or Nix attribute sets) for each compiler call. Each generated Nix flake should represent a single build step that, when evaluated, would execute the corresponding Rust compiler command with the specified arguments, environment, and working directory. - * **Organize Generated Flakes:** Store these generated flakes in a structured manner, e.g., `test-rust2/bootstrap/step001/flake.nix`, `test-rust2/bootstrap/step002/flake.nix`, etc., to represent the sequence of build steps. - -3. **Address `src/ci/channel` Panic (Likely Bypassed)** - * This issue, previously a critical blocker, will likely be bypassed by the new approach. Since `bootstrap.py` will no longer execute the Rust binary that panics on `src/ci/channel`, the direct cause of this panic will be removed. - * However, if `bootstrap.py` itself needs to read this file for its internal logic (e.g., to determine the channel for emitting the Nix expression), we might still need to ensure its accessibility or provide a default value. - -4. **Generalize `rust_root` and `build_dir` Handling (Medium Priority)** - * **Problem:** `x.py` (and `bootstrap.py`) expects to be run from the root of the Rust source tree, but needs to perform writable operations in a temporary directory. - * **Solution:** Ensure `bootstrap.py` correctly identifies and uses the read-only source root (`$src`) and the writable build directory (`$TMPDIR`). This might involve modifying `bootstrap.py` to accept parameters for these paths or to derive them robustly within the Nix build environment. - -5. **Clean Up `flake.nix` (Low Priority)** - * Remove all debug `echo` and `ls` commands from the `buildPhase` once the core functionality is working. - * Remove `RUST_BACKTRACE=full` once the panic is resolved (which should be the case with the new approach). - -## Current Status - -* The `task.md` file has been updated with the new strategy. -* The previous attempts to debug the `fs::read_to_string` panic in the Rust binary are now superseded by the goal of preventing its execution entirely. - -## Next Immediate Step - -Modify the `run` function in `test-rust2/standalonex/src/bootstrap/bootstrap.py` to emit JSON descriptions of compiler invocations instead of executing them. This is the foundational step for transforming `x.py` into a Nix expression generator. We will need to carefully consider the structure of the JSON output to make it easily consumable by Nix. We will also need to ensure that `sys.exit(0)` is called after printing the JSON to prevent further execution within `bootstrap.py`. \ No newline at end of file diff --git a/standalonex/test_minimal/Cargo.lock b/standalonex/test_minimal/Cargo.lock new file mode 100644 index 00000000..919b2c43 --- /dev/null +++ b/standalonex/test_minimal/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "hello-rust" +version = "0.1.0" diff --git a/standalonex/test_minimal/Cargo.nix b/standalonex/test_minimal/Cargo.nix new file mode 100644 index 00000000..1d122a7c --- /dev/null +++ b/standalonex/test_minimal/Cargo.nix @@ -0,0 +1,68 @@ +# This file was @generated by cargo2nix 0.12.0. +# It is not intended to be manually edited. + +args@{ release ? true +, rootFeatures ? [ + "hello-rust/default" + ] +, rustPackages +, buildRustPackages +, hostPlatform +, hostPlatformCpu ? null +, hostPlatformFeatures ? [ ] +, target ? null +, codegenOpts ? null +, profileOpts ? null +, cargoUnstableFlags ? null +, rustcLinkFlags ? null +, rustcBuildFlags ? null +, mkRustCrate +, rustLib +, lib +, workspaceSrc +, ignoreLockHash +, cargoConfig ? { } +, +}: +let + nixifiedLockHash = "7ba833d7ed5962f631d3832c05cf774952db7c0b6e731047aef00a56b322ff2e"; + workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; + currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); + lockHashIgnored = + if ignoreLockHash + then builtins.trace "Ignoring lock hash" ignoreLockHash + else ignoreLockHash; +in +if !lockHashIgnored && (nixifiedLockHash != currentLockHash) then + throw ("Cargo.nix ${nixifiedLockHash} is out of sync with Cargo.lock ${currentLockHash}") +else + let + inherit (rustLib) fetchCratesIo fetchCrateLocal fetchCrateGit fetchCrateAlternativeRegistry expandFeatures decideProfile genDrvsByProfile; + cargoConfig' = if cargoConfig != { } then cargoConfig else + if builtins.pathExists ./.cargo/config then lib.importTOML ./.cargo/config else + if builtins.pathExists ./.cargo/config.toml then lib.importTOML ./.cargo/config.toml else { }; + profilesByName = { }; + rootFeatures' = expandFeatures rootFeatures; + overridableMkRustCrate = f: + let + drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ + inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts cargoUnstableFlags rustcLinkFlags rustcBuildFlags; + cargoConfig = cargoConfig'; + } // (f profileName))); + in + { compileMode ? null, profileName ? decideProfile compileMode release }: + let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; }; + in + { + cargo2nixVersion = "0.12.0"; + workspace = { + hello-rust = rustPackages.unknown.hello-rust."0.1.0"; + }; + "unknown".hello-rust."0.1.0" = overridableMkRustCrate (profileName: rec { + name = "hello-rust"; + version = "0.1.0"; + registry = "unknown"; + src = fetchCrateLocal workspaceSrc; + }); + + } diff --git a/standalonex/test_minimal/flake.nix b/standalonex/test_minimal/flake.nix new file mode 100644 index 00000000..b4113691 --- /dev/null +++ b/standalonex/test_minimal/flake.nix @@ -0,0 +1,53 @@ +{ + description = "A minimal Rust project using cargo2nix"; + + inputs = { + nixpkgs.url = "github:meta-introspector/nixpkgs?ref=feature/CRQ-016-nixify"; + flake-utils.url = "github:meta-introspector/flake-utils?ref=feature/CRQ-016-nixify"; + rust-overlay.url = "github:meta-introspector/rust-overlay?ref=feature/CRQ-016-nixify"; + cargo2nix.url = "github:meta-introspector/cargo2nix?ref=feature/CRQ-016-nixify"; + }; + + outputs = { self, nixpkgs, flake-utils, rust-overlay, cargo2nix }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ + rust-overlay.overlays.default + cargo2nix.overlays.default + ]; + }; + + rustVersion = "1.75.0"; # Explicitly set rust version + rustPkgs = pkgs.rustBuilder.makePackageSet { + inherit rustVersion; + packageFun = import ./Cargo.nix; + workspaceSrc = ./.; + }; + + helloRustApp = rustPkgs.workspace.hello-rust { }; # Call the function to get the derivation + in + { + packages = { + hello-rust = helloRustApp; + default = helloRustApp; + }; + + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.rust-bin.stable.${rustVersion}.default + pkgs.cargo + pkgs.rustc + pkgs.rustfmt + pkgs.clippy + pkgs.git + pkgs.pkg-config + pkgs.cmake + pkgs.libiconv # For macOS + ]; + CARGO_HOME = "${pkgs.writeText "cargo-home" ""}"; + RUST_SRC_PATH = "${pkgs.rustPlatform.rustLibSrc}"; + }; + }); +} diff --git a/standalonex/test_minimal/src/main.rs b/standalonex/test_minimal/src/main.rs new file mode 100644 index 00000000..66449d79 --- /dev/null +++ b/standalonex/test_minimal/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, Rust!"); +} diff --git a/standalonex/wrap_rust.py b/standalonex/wrap_rust.py deleted file mode 100644 index b70514cd..00000000 --- a/standalonex/wrap_rust.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import json -import subprocess - -# Counter for flake steps -flake_step_counter = 0 - -def generate_nix_flake(command, args, env): - global flake_step_counter - flake_step_counter += 1 - step_dir = f"test-rust2/bootstrap/step{flake_step_counter:03d}" - os.makedirs(step_dir, exist_ok=True) - - flake_content = f""" -{{ - description = "Nix flake for Rust compiler command step {flake_step_counter}"; - - outputs = {{ self, nixpkgs }}: - let - pkgs = import nixpkgs {{ system = "aarch64-linux"; }}; # Assuming aarch64-linux - in - {{ - packages.aarch64-linux.default = pkgs.stdenv.mkDerivation {{ - pname = "rust-compiler-step-{flake_step_counter:03d}"; - version = "0.1.0"; - - # Capture command, args, and environment - buildCommand = ''' - echo "Command: {command}" - echo "Args: {args}" - echo "Env: {json.dumps(env)}" - # TODO: Reconstruct and execute the actual command here - '''; - }}; - }}; -}} -""" - with open(os.path.join(step_dir, "flake.nix"), "w") as f: - f.write(flake_content) - -def main(): - original_command = sys.argv[1] - original_args = sys.argv[2:] - original_env = os.environ.copy() - - # Generate Nix flake for this command - generate_nix_flake(original_command, original_args, original_env) - - # Execute the original command - # This part needs to be careful to avoid infinite recursion if original_command is also wrap_rust.py - # For now, we'll assume original_command is the actual rustc/cargo - subprocess.run([original_command] + original_args, env=original_env) - -if __name__ == "__main__": - main() diff --git a/standalonex/x.py b/standalonex/x.py deleted file mode 100755 index 6c4c1068..00000000 --- a/standalonex/x.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# Some systems don't have `python3` in their PATH. This isn't supported by x.py directly; -# they should use `x` or `x.ps1` instead. - -# This file is only a "symlink" to bootstrap.py, all logic should go there. - -# Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point -# must use the normal `if __name__ == '__main__':` convention to avoid problems. -if __name__ == '__main__': - import os - import sys - import warnings - from inspect import cleandoc - - major = sys.version_info.major - minor = sys.version_info.minor - - # If this is python2, check if python3 is available and re-execute with that - # interpreter. Only python3 allows downloading CI LLVM. - # - # This matters if someone's system `python` is python2. - if major < 3: - try: - os.execvp("py", ["py", "-3"] + sys.argv) - except OSError: - try: - os.execvp("python3", ["python3"] + sys.argv) - except OSError: - # Python 3 isn't available, fall back to python 2 - pass - - # soft deprecation of old python versions - skip_check = os.environ.get("RUST_IGNORE_OLD_PYTHON") == "1" - if not skip_check and (major < 3 or (major == 3 and minor < 6)): - msg = cleandoc(""" - Using python {}.{} but >= 3.6 is recommended. Your python version - should continue to work for the near future, but this will - eventually change. If python >= 3.6 is not available on your system, - please file an issue to help us understand timelines. - - This message can be suppressed by setting `RUST_IGNORE_OLD_PYTHON=1` - """.format(major, minor)) - warnings.warn(msg, stacklevel=1) - - rust_dir = os.path.dirname(os.path.abspath(__file__)) - # For the import below, have Python search in src/bootstrap first. - sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap")) - - import bootstrap - bootstrap.main() diff --git a/temp_shell.nix b/temp_shell.nix new file mode 100644 index 00000000..a2e45832 --- /dev/null +++ b/temp_shell.nix @@ -0,0 +1,8 @@ +{ pkgs ? import (builtins.getFlake "github:meta-introspector/nixpkgs?rev=26833ad1dad83826ef7cc52e0009ca9b7097c79f") { } }: + +pkgs.mkShell { + buildInputs = [ + pkgs.rustc + pkgs.cargo + ]; +} diff --git a/temp_stage0_Cargo.toml b/temp_stage0_Cargo.toml new file mode 100644 index 00000000..3af26482 --- /dev/null +++ b/temp_stage0_Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "stage0" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = "1" +serde_derive = "1" diff --git a/test-flake/flake.nix b/test-flake/flake.nix new file mode 100644 index 00000000..a001abda --- /dev/null +++ b/test-flake/flake.nix @@ -0,0 +1,8 @@ +{ + description = "Minimal test flake"; + + outputs = { self }: + { + packages.aarch64-linux.default = "Hello, Nix!"; + }; +} diff --git a/test.sh b/test.sh index 73005a44..9c6e3e08 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,8 @@ #!/bin/sh -echo "--- Setting up Test Environment (Replicating Nix Build) ---" +set -euo pipefail -# Hardcoded Nix store paths (obtained from previous nix eval) -SCCACHE_PATH="/nix/store/wnb0ak90fv19ys0hxzb9c2jkfdshys1a-sccache-0.10.0" -CURL_PATH="/nix/store/l93sz0xmd0pm4xjyz9bhlyfgh09bzsxn-curl-8.14.1-bin" -RUSTC_PATH="/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/rustc" -CARGO_PATH="/nix/store/i7yprsq7l6zi19954b8lxcd5ibxkp14j-rust-legacy-1.92.0-nightly-2025-10-16/bin/cargo" -GREP_PATH="/nix/store/v2i6say53v0al0s4gv7kpq178wp73qyl-gnugrep-3.12/bin/grep" # Assuming grep is available in Nixpkgs +echo "--- Setting up Test Environment (Replicating Nix Build) ---" # Define paths and variables (replicating Nix build environment) TMPDIR=$(mktemp -d) @@ -18,47 +13,13 @@ export CARGO_WORKSPACE_ROOT="/nonexistent/workspace/root" # Attempt to trick car mkdir -p "$CARGO_HOME" mkdir -p "$CARGO_TARGET_DIR" -# Add sccache and grep to PATH for immediate use in the script -export PATH="${SCCACHE_PATH}/bin:${GREP_PATH}:${PATH}" -#/nix/store/wnb0ak90fv19ys0hxzb9c2jkfdshys1a-sccache-0.10.0/bin/sccache - -# Replicating preConfigure logic -#export RUSTC_WRAPPER="${SCCACHE_PATH}/bin/sccache" -#export SCCACHE_DIR="$TMPDIR/sccache" -#export SCCACHE_TEMPDIR="$TMPDIR/sccache-tmp" -#mkdir -p "$SCCACHE_DIR" -#mkdir -p "$SCCACHE_TEMPDIR" -#env -#sccache --stop-server || true # Stop any existing sccache server -#sccache --start-server - -export PATH="${CURL_PATH}/bin:$PATH" -export CURL="${CURL_PATH}/bin/curl" - -# Replicating buildPhase logic -CONFIG_TOML_PATH="./config.toml" -COMPILER_DATE=$(${GREP_PATH} -oP 'compiler_date=\K.*' /data/data/com.termux.nix/files/home/pick-up-nix2/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/src/stage0) # Using absolute path for src/stage0 - -cat > "$CONFIG_TOML_PATH" < "${BIN_ROOT}/.rustc-stamp" - -# Run the build command -pushd /data/data/com.termux.nix/files/home/nix/vendor/rust/platform-tools-agave-rust-solana/vendor/rust-src/ -python .//x.py build -vv -- --no-default-features --features "" -popd +# Determine the current flake's GitHub reference dynamically if possible, +# or use a hardcoded one for now as per user's instruction. +# For this specific case, the user provided the exact reference. +FLAKE_REF="github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex" -#echo "--- Stopping sccache server ---" -#sccache --stop-server # Ensure sccache server is stopped after build +# Run the build command within the Nix shell +nix shell "$FLAKE_REF#devShells.aarch64-linux.default" --command "./build_rust_bootstrap.sh" echo "--- Cleaning up temporary directory ---" rm -rf "$TMPDIR" diff --git a/test_nix_preconditions.sh b/test_nix_preconditions.sh new file mode 100755 index 00000000..037ade30 --- /dev/null +++ b/test_nix_preconditions.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "--- Testing Preconditions for Nix Flake Build ---" +echo "" + +# --- Helper function for logging --- +log_success() { + echo "✅ SUCCESS: $1" +} + +log_failure() { + echo "❌ FAILURE: $1" + exit 1 +} + +log_info() { + echo "ℹī¸ INFO: $1" +} + +# --- Precondition 1: Verify Nix command availability --- +log_info "1. Verifying 'nix' command availability..." +if command -v nix &> /dev/null; then + log_success "Nix command found." +else + log_failure "Nix command not found. Please install Nix." +fi +echo "" + +# --- Precondition 2: Verify Rust toolchain sysroot exists --- +log_info "2. Verifying Rust toolchain sysroot for pkgs.rust-bin.stable.\"1.84.1\".default..." + +RUST_TOOLCHAIN_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" --expr ' + let + standalonexFlake = builtins.getFlake "github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex"; + pkgs = import standalonexFlake.inputs.nixpkgs { + system = "aarch64-linux"; # Explicitly set system + overlays = [ standalonexFlake.inputs.rustOverlay.overlays.default ]; + }; + in + pkgs.rustPlatform.rustLibSrc # Access the rustLibSrc attribute +') + +if [[ -d "$RUST_TOOLCHAIN_PATH" ]]; then + log_success "Rust toolchain sysroot found at: $RUST_TOOLCHAIN_PATH/lib/rustlib/src/rust" +else + log_failure "Rust toolchain sysroot NOT found at: $RUST_TOOLCHAIN_PATH/lib/rustlib/src/rust" +fi +echo "" + +# --- Precondition 3: Verify Rust source flake (rustSrcFlake) exists --- +log_info "3. Verifying Rust source flake (rustSrcFlake) exists..." + +RUST_SRC_FLAKE_PATH=$(nix eval --raw --extra-experimental-features "nix-command flakes" --expr ' + let + standalonexFlake = builtins.getFlake "github:meta-introspector/rust-bootstrap-nix?rev=be3f35712b133efd47073a3a45203ddca533fe01&dir=standalonex"; + in + standalonexFlake.inputs.rustSrcFlake.outPath +') + +if [[ -d "$RUST_SRC_FLAKE_PATH" ]]; then + log_success "Rust source flake found at: $RUST_SRC_FLAKE_PATH" + # Further check for a known file within the source + if [[ -f "$RUST_SRC_FLAKE_PATH/src/ci/channel" ]]; then + log_success "Known file 'src/ci/channel' found within Rust source flake." + else + log_failure "Known file 'src/ci/channel' NOT found within Rust source flake. Path might be incorrect or incomplete." + fi +else + log_failure "Rust source flake NOT found at: $RUST_SRC_FLAKE_PATH" +fi +echo "" + +echo "--- All Precondition Tests Complete ---" \ No newline at end of file diff --git a/update_all_flakes.sh b/update_all_flakes.sh new file mode 100755 index 00000000..e53cd3cd --- /dev/null +++ b/update_all_flakes.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Define the base directory (current working directory) +BASE_DIR=$(pwd) + +# List of flake directories relative to BASE_DIR +FLAKE_DIRS=( + "flakes/config" + "flakes/xpy-json-output-flake" + "flakes/json-processor" + "flakes/json-processor-flake" + "flakes/evaluate-rust" + "." # The current directory itself + "standalonex" +) + +LOG_FILE="nix_flake_update_$(date +%Y%m%d_%H%M%S).log" + +echo "Starting Nix flake updates..." | tee -a "$LOG_FILE" +echo "Logging all output to: $LOG_FILE" | tee -a "$LOG_FILE" + +for dir in "${FLAKE_DIRS[@]}"; do + FULL_PATH="$BASE_DIR/$dir" + if [ -d "$FULL_PATH" ]; then + echo "----------------------------------------------------" | tee -a "$LOG_FILE" + echo "Processing flake in: $FULL_PATH" | tee -a "$LOG_FILE" + + # --- Pre-update grep for url --- + echo "--- Pre-update grep for 'url' in flake.lock ---" | tee -a "$LOG_FILE" + (cd "$FULL_PATH" && grep "url" flake.lock 2>/dev/null) | tee -a "$LOG_FILE" + if [ $? -ne 0 ]; then + echo "No 'url' found in flake.lock (or flake.lock does not exist)." | tee -a "$LOG_FILE" + fi + + # --- Pre-update grep for NixOS --- + echo "--- Pre-update grep for 'NixOS' in flake.lock ---" | tee -a "$LOG_FILE" + (cd "$FULL_PATH" && grep -i "NixOS" flake.lock 2>/dev/null) | tee -a "$LOG_FILE" + if [ $? -ne 0 ]; then + echo "No 'NixOS' found in flake.lock (or flake.lock does not exist)." | tee -a "$LOG_FILE" + fi + + # --- Run nix flake update --- + echo "--- Running nix flake update --verbose ---" | tee -a "$LOG_FILE" + ( + cd "$FULL_PATH" || exit + timeout 10s nix flake update --verbose 2>&1 | tee -a "$LOG_FILE" + ) + UPDATE_STATUS=$? + if [ $UPDATE_STATUS -ne 0 ]; then + echo "Error updating flake in $FULL_PATH. Exit code: $UPDATE_STATUS. Continuing..." | tee -a "$LOG_FILE" + fi + + # --- Post-update grep for url --- + echo "--- Post-update grep for 'url' in flake.lock ---" | tee -a "$LOG_FILE" + (cd "$FULL_PATH" && grep "url" flake.lock 2>/dev/null) | tee -a "$LOG_FILE" + if [ $? -ne 0 ]; then + echo "No 'url' found in flake.lock (or flake.lock does not exist)." | tee -a "$LOG_FILE" + fi + + # --- Post-update grep for NixOS --- + echo "--- Post-update grep for 'NixOS' in flake.lock ---" | tee -a "$LOG_FILE" + (cd "$FULL_PATH" && grep -i "NixOS" flake.lock 2>/dev/null) | tee -a "$LOG_FILE" + if [ $? -ne 0 ]; then + echo "No 'NixOS' found in flake.lock (or flake.lock does not exist)." | tee -a "$LOG_FILE" + fi + + else + echo "Warning: Directory $FULL_PATH not found. Skipping." | tee -a "$LOG_FILE" + fi +done + +echo "----------------------------------------------------" | tee -a "$LOG_FILE" +echo "All flake updates attempted." | tee -a "$LOG_FILE" \ No newline at end of file