From abf10fdccf5feb387b329e1835e3b1b9394cd785 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Fri, 1 May 2026 12:58:18 -0700 Subject: [PATCH 01/50] Remove WORKSPACE support (#4005) closes https://github.com/bazelbuild/rules_rust/issues/3818 --- tools/rust_analyzer/BUILD.bazel | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/rust_analyzer/BUILD.bazel b/tools/rust_analyzer/BUILD.bazel index dbc016a44d..965eb6a964 100644 --- a/tools/rust_analyzer/BUILD.bazel +++ b/tools/rust_analyzer/BUILD.bazel @@ -2,6 +2,18 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//rust:defs.bzl", "rust_binary", "rust_clippy", "rust_library", "rust_test") load("//tools/private:tool_utils.bzl", "aspect_repository") +exports_files( + [ + "aquery.rs", + "bin/discover_rust_project.rs", + "bin/gen_rust_project.rs", + "bin/validate.rs", + "lib.rs", + "rust_project.rs", + ], + visibility = ["//visibility:public"], +) + rust_binary( name = "discover_bazel_rust_project", srcs = ["bin/discover_rust_project.rs"], From 39efdc40987ca68c56022253d36c527d184488a1 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Fri, 1 May 2026 18:33:37 -0400 Subject: [PATCH 02/50] feat: Add BPF triple constraint mapping (#3696) Add support for tier 3 targets bpfeb-unknown-none and bpfel-unknown-none (see https://github.com/rust-lang/rust/blob/f5e2df7/src/doc/rustc/src/platform-support.md?plain=1#L311-L312). This is modeled after https://github.com/bazelbuild/rules_rust/pull/3507 and should probably be updated if/when https://github.com/bazelbuild/platforms/pull/131 is merged. (please use rebase merge when landing this as the proper commit message is in the commit, rather than the PR description) /cc @avrabe From 52c68afa50c0c1f2748d9832693d83e35a06e5cb Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 20 Jan 2026 10:31:15 -0500 Subject: [PATCH 03/50] Rearrange link paths on Windows to reduce size overruns and fix errors --- util/process_wrapper/main.rs | 186 ++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 39a6d6db16..7ffd66a8c1 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -19,10 +19,15 @@ mod rustc; mod util; use std::collections::HashMap; +#[cfg(windows)] +use std::collections::HashSet; use std::fmt; -use std::fs::{copy, OpenOptions}; +use std::fs::{self, copy, OpenOptions}; use std::io; +use std::path::PathBuf; use std::process::{exit, Command, ExitStatus, Stdio}; +#[cfg(windows)] +use std::time::{SystemTime, UNIX_EPOCH}; use tinyjson::JsonValue; @@ -73,6 +78,175 @@ macro_rules! debug_log { }; } +#[cfg(windows)] +struct TemporaryDirectoryGuard { + path: Option, +} + +#[cfg(windows)] +impl TemporaryDirectoryGuard { + fn new(path: Option) -> Self { + Self { path } + } + + fn take(&mut self) -> Option { + self.path.take() + } +} + +#[cfg(windows)] +impl Drop for TemporaryDirectoryGuard { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + let _ = fs::remove_dir_all(path); + } + } +} + +#[cfg(not(windows))] +struct TemporaryDirectoryGuard; + +#[cfg(not(windows))] +impl TemporaryDirectoryGuard { + fn new(_: Option) -> Self { + TemporaryDirectoryGuard + } + + fn take(&mut self) -> Option { + None + } +} + +#[cfg(windows)] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + let mut dependency_paths = Vec::new(); + let mut filtered_args = Vec::with_capacity(args.len()); + + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + if arg == "-L" { + if let Some(next) = args.get(i + 1) { + if let Some(path) = next.strip_prefix("dependency=") { + dependency_paths.push(PathBuf::from(path)); + i += 2; + continue; + } + } + } + + if let Some(path) = arg.strip_prefix("-Ldependency=") { + dependency_paths.push(PathBuf::from(path)); + i += 1; + continue; + } + + filtered_args.push(arg.clone()); + i += 1; + } + + if dependency_paths.is_empty() { + return Ok((filtered_args, None)); + } + + let unique_suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let dir_name = format!( + "rules_rust_process_wrapper_deps_{}_{}", + std::process::id(), + unique_suffix + ); + + let base_dir = std::env::current_dir().map_err(|e| { + ProcessWrapperError(format!("unable to read current working directory: {}", e)) + })?; + let unified_dir = base_dir.join(&dir_name); + fs::create_dir_all(&unified_dir).map_err(|e| { + ProcessWrapperError(format!( + "unable to create unified dependency directory {}: {}", + unified_dir.display(), + e + )) + })?; + + let mut seen = HashSet::new(); + for path in dependency_paths { + let entries = fs::read_dir(&path).map_err(|e| { + ProcessWrapperError(format!( + "unable to read dependency search path {}: {}", + path.display(), + e + )) + })?; + + for entry in entries { + let entry = entry.map_err(|e| { + ProcessWrapperError(format!( + "unable to iterate dependency search path {}: {}", + path.display(), + e + )) + })?; + let file_type = entry.file_type().map_err(|e| { + ProcessWrapperError(format!( + "unable to inspect dependency search path {}: {}", + path.display(), + e + )) + })?; + if !(file_type.is_file() || file_type.is_symlink()) { + continue; + } + + let file_name = entry.file_name(); + let file_name_lower = file_name + .to_string_lossy() + .to_ascii_lowercase(); + if !seen.insert(file_name_lower) { + continue; + } + + let dest = unified_dir.join(&file_name); + let src = entry.path(); + match fs::hard_link(&src, &dest) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(err) => { + debug_log!( + "failed to hardlink {} to {} ({}), falling back to copy", + src.display(), + dest.display(), + err + ); + fs::copy(&src, &dest).map_err(|copy_err| { + ProcessWrapperError(format!( + "unable to copy {} into unified dependency dir {}: {}", + src.display(), + dest.display(), + copy_err + )) + })?; + } + } + } + } + + filtered_args.push(format!("-Ldependency={}", unified_dir.display())); + + Ok((filtered_args, Some(unified_dir))) +} + +#[cfg(not(windows))] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + Ok((args.to_vec(), None)) +} + fn json_warning(line: &str) -> JsonValue { JsonValue::Object(HashMap::from([ ( @@ -120,9 +294,13 @@ fn process_line( fn main() -> Result<(), ProcessWrapperError> { let opts = options().map_err(|e| ProcessWrapperError(e.to_string()))?; + let (child_arguments, dep_dir_cleanup) = + consolidate_dependency_search_paths(&opts.child_arguments)?; + let mut temp_dir_guard = TemporaryDirectoryGuard::new(dep_dir_cleanup); + let mut command = Command::new(opts.executable); command - .args(opts.child_arguments) + .args(child_arguments) .env_clear() .envs(opts.child_environment) .stdout(if let Some(stdout_file) = opts.stdout_file { @@ -228,6 +406,10 @@ fn main() -> Result<(), ProcessWrapperError> { } } + if let Some(path) = temp_dir_guard.take() { + let _ = fs::remove_dir_all(path); + } + exit(code) } From 09804e850eff555caab874b36915d7391bcf9224 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 6 Feb 2026 20:00:57 -0500 Subject: [PATCH 04/50] Revert "Fix stamping for rules that don't have a stamp attribute (#3829)" This reverts commit f198ddee7f49ac351d27204b1488df0af2512fac. --- rust/private/stamp.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/stamp.bzl b/rust/private/stamp.bzl index bff7cbadf3..a05c255c9a 100644 --- a/rust/private/stamp.bzl +++ b/rust/private/stamp.bzl @@ -10,7 +10,7 @@ def is_stamping_enabled(ctx, attr): Returns: bool: The stamp value """ - stamp_num = getattr(attr, "stamp", 0) + stamp_num = getattr(attr, "stamp", -1) if stamp_num == 1: return True elif stamp_num == 0: From 8ce8b249931b0e78e7cc2387adda4ee0a2babfd9 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 6 Feb 2026 20:01:07 -0500 Subject: [PATCH 05/50] Revert "Switch stamping detection to ctx.configuration.stamp_binaries() (#3816)" This reverts commit 9586468d1eaa8d22f527966cb5f43a4870463649. --- rust/private/BUILD.bazel | 3 ++ rust/private/rust.bzl | 4 +++ rust/private/rustc.bzl | 2 +- rust/private/stamp.bzl | 62 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index 89444bb9bc..d18e895493 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//rust/private:rust_analyzer.bzl", "rust_analyzer_detect_sysroot") load("//rust/private:rustc.bzl", "is_proc_macro_dep", "is_proc_macro_dep_enabled") +load("//rust/private:stamp.bzl", "stamp_build_setting") # Exported for docs exports_files(["providers.bzl"]) @@ -32,6 +33,8 @@ bzl_library( ], ) +stamp_build_setting(name = "stamp") + # This setting may be used to identify dependencies of proc-macro-s. # This feature is only enabled if `is_proc_macro_dep_enabled` is true. # Its value controls the BAZEL_RULES_RUST_IS_PROC_MACRO_DEP environment variable diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 9d9c1d12b4..301eec048e 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -821,6 +821,10 @@ _COMMON_ATTRS = { doc = "Enable collection of cfg flags with results stored in CrateInfo.cfgs.", default = Label("//rust/settings:collect_cfgs"), ), + "_stamp_flag": attr.label( + doc = "A setting used to determine whether or not the `--stamp` flag is enabled", + default = Label("//rust/private:stamp"), + ), } | RUSTC_ATTRS | RUSTC_ALLOCATOR_LIBRARIES_ATTRS _PLATFORM_ATTRS = { diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index ad84f1066f..c0fa573683 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1541,7 +1541,7 @@ def rustc_compile_action( linkstamps = depset([]) # Determine if the build is currently running with --stamp - stamp = is_stamping_enabled(ctx, attr) + stamp = is_stamping_enabled(attr) # Add flags for any 'rustc' lints that are specified. # diff --git a/rust/private/stamp.bzl b/rust/private/stamp.bzl index a05c255c9a..1a7cab65cc 100644 --- a/rust/private/stamp.bzl +++ b/rust/private/stamp.bzl @@ -1,10 +1,63 @@ -"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled""" +"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled -def is_stamping_enabled(ctx, attr): +This module can be removed likely after the following PRs ar addressed: +- https://github.com/bazelbuild/bazel/issues/11164 +""" + +load("//rust/private:utils.bzl", "dedent") + +StampSettingInfo = provider( + doc = "Information about the `--stamp` command line flag", + fields = { + "value": "bool: Whether or not the `--stamp` flag was enabled", + }, +) + +def _stamp_build_setting_impl(ctx): + return StampSettingInfo(value = ctx.attr.value) + +_stamp_build_setting = rule( + doc = dedent("""\ + Whether to encode build information into the binary. Possible values: + + - stamp = 1: Always stamp the build information into the binary, even in [--nostamp][stamp] builds. \ + This setting should be avoided, since it potentially kills remote caching for the binary and \ + any downstream actions that depend on it. + - stamp = 0: Always replace build information by constant values. This gives good build result caching. + - stamp = -1: Embedding of build information is controlled by the [--[no]stamp][stamp] flag. + + Stamped binaries are not rebuilt unless their dependencies change. + [stamp]: https://docs.bazel.build/versions/main/user-manual.html#flag--stamp + """), + implementation = _stamp_build_setting_impl, + attrs = { + "value": attr.bool( + doc = "The default value of the stamp build flag", + mandatory = True, + ), + }, +) + +def stamp_build_setting(name, visibility = ["//visibility:public"]): + native.config_setting( + name = "stamp_detect", + values = {"stamp": "1"}, + visibility = visibility, + ) + + _stamp_build_setting( + name = name, + value = select({ + ":stamp_detect": True, + "//conditions:default": False, + }), + visibility = visibility, + ) + +def is_stamping_enabled(attr): """Determine whether or not build stamping is enabled Args: - ctx (ctx): The rule's context object attr (struct): A rule's struct of attributes (`ctx.attr`) Returns: @@ -16,6 +69,7 @@ def is_stamping_enabled(ctx, attr): elif stamp_num == 0: return False elif stamp_num == -1: - return ctx.configuration.stamp_binaries() + stamp_flag = getattr(attr, "_stamp_flag", None) + return stamp_flag[StampSettingInfo].value if stamp_flag else False else: fail("Unexpected `stamp` value: {}".format(stamp_num)) From 2c5c1dce24e0e5f4d295d1aca70b7606b507b226 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 16 Feb 2026 15:50:51 -0500 Subject: [PATCH 06/50] Fix process-wrapper link lib handling when using argfiles --- util/process_wrapper/main.rs | 73 +++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 7ffd66a8c1..5d057b08cf 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -20,7 +20,7 @@ mod util; use std::collections::HashMap; #[cfg(windows)] -use std::collections::HashSet; +use std::collections::{HashSet, VecDeque}; use std::fmt; use std::fs::{self, copy, OpenOptions}; use std::io; @@ -34,6 +34,8 @@ use tinyjson::JsonValue; use crate::options::options; use crate::output::{process_output, LineOutput}; use crate::rustc::ErrorFormat; +#[cfg(windows)] +use crate::util::read_file_to_array; #[cfg(windows)] fn status_code(status: ExitStatus, was_killed: bool) -> i32 { @@ -118,35 +120,64 @@ impl TemporaryDirectoryGuard { } #[cfg(windows)] -fn consolidate_dependency_search_paths( - args: &[String], -) -> Result<(Vec, Option), ProcessWrapperError> { +fn get_dependency_search_paths_from_args( + initial_args: &[String], +) -> Result<(Vec, Vec), ProcessWrapperError> { let mut dependency_paths = Vec::new(); - let mut filtered_args = Vec::with_capacity(args.len()); + let mut filtered_args = Vec::new(); + let mut argfile_contents: HashMap> = HashMap::new(); + + let mut queue: VecDeque<(String, Option)> = initial_args + .iter() + .map(|arg| (arg.clone(), None)) + .collect(); + + while let Some((arg, parent_argfile)) = queue.pop_front() { + let target = match &parent_argfile { + Some(p) => argfile_contents.entry(format!("{}.filtered", p)).or_default(), + None => &mut filtered_args, + }; - let mut i = 0; - while i < args.len() { - let arg = &args[i]; if arg == "-L" { - if let Some(next) = args.get(i + 1) { - if let Some(path) = next.strip_prefix("dependency=") { - dependency_paths.push(PathBuf::from(path)); - i += 2; - continue; - } + let next_arg = queue.front().map(|(a, _)| a.as_str()); + if let Some(path) = next_arg.and_then(|n| n.strip_prefix("dependency=")) { + dependency_paths.push(PathBuf::from(path)); + queue.pop_front(); + } else { + target.push(arg); } - } - - if let Some(path) = arg.strip_prefix("-Ldependency=") { + } else if let Some(path) = arg.strip_prefix("-Ldependency=") { dependency_paths.push(PathBuf::from(path)); - i += 1; - continue; + } else if let Some(argfile_path) = arg.strip_prefix('@') { + let lines = read_file_to_array(argfile_path).map_err(|e| { + ProcessWrapperError(format!("unable to read argfile {}: {}", argfile_path, e)) + })?; + + for line in lines { + queue.push_back((line, Some(argfile_path.to_string()))); + } + + target.push(format!("@{}.filtered", argfile_path)); + } else { + target.push(arg); } + } - filtered_args.push(arg.clone()); - i += 1; + for (path, content) in argfile_contents { + fs::write(&path, content.join("\n")).map_err(|e| { + ProcessWrapperError(format!("unable to write filtered argfile {}: {}", path, e)) + })?; } + Ok((dependency_paths, filtered_args)) +} + +#[cfg(windows)] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + let (dependency_paths, mut filtered_args) = get_dependency_search_paths_from_args(args)?; + if dependency_paths.is_empty() { return Ok((filtered_args, None)); } From 8594a6b8c8adb0fefeecf5926392272674533962 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 18 Feb 2026 05:12:26 -0500 Subject: [PATCH 07/50] Rewrite process_wrapper_bootstrap to cc --- rust/settings/BUILD.bazel | 3 - rust/settings/settings.bzl | 9 - test/process_wrapper_bootstrap/BUILD.bazel | 20 +- .../bootstrap_process_wrapper_probe.rs | 10 + .../bootstrap_process_wrapper_test.rs | 60 ++++-- .../process_wrapper_bootstrap_test.bzl | 78 -------- util/process_wrapper/BUILD.bazel | 8 +- util/process_wrapper/private/BUILD.bazel | 11 +- .../private/bootstrap_process_wrapper.bzl | 73 -------- .../private/bootstrap_process_wrapper.cc | 173 ++++++++++++++++++ .../private/process_wrapper.bat | 40 ---- .../private/process_wrapper.sh | 47 ----- 12 files changed, 252 insertions(+), 280 deletions(-) create mode 100644 test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs delete mode 100644 test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl delete mode 100644 util/process_wrapper/private/bootstrap_process_wrapper.bzl create mode 100644 util/process_wrapper/private/bootstrap_process_wrapper.cc delete mode 100755 util/process_wrapper/private/process_wrapper.bat delete mode 100755 util/process_wrapper/private/process_wrapper.sh diff --git a/rust/settings/BUILD.bazel b/rust/settings/BUILD.bazel index d2b84debcc..6f0cd9dbca 100644 --- a/rust/settings/BUILD.bazel +++ b/rust/settings/BUILD.bazel @@ -18,7 +18,6 @@ load( "experimental_use_cc_common_link", "experimental_use_coverage_metadata_files", "experimental_use_global_allocator", - "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", "extra_exec_rustc_env", "extra_exec_rustc_flag", "extra_exec_rustc_flags", @@ -94,8 +93,6 @@ experimental_use_global_allocator() experimental_use_allocator_libraries_with_mangled_symbols() -experimental_use_sh_toolchain_for_bootstrap_process_wrapper() - extra_exec_rustc_env() extra_exec_rustc_flag() diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index 2769272dc5..b6d1c0aea4 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -265,15 +265,6 @@ def experimental_link_std_dylib(): build_setting_default = False, ) -def experimental_use_sh_toolchain_for_bootstrap_process_wrapper(): - """A flag to control whether the shell path from a shell toolchain (`@bazel_tools//tools/sh:toolchain_type`) \ - is embedded into the bootstrap process wrapper for the `.sh` file. - """ - bool_flag( - name = "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", - build_setting_default = False, - ) - def toolchain_linker_preference(): """A flag to control which linker is preferred for linking Rust binaries. diff --git a/test/process_wrapper_bootstrap/BUILD.bazel b/test/process_wrapper_bootstrap/BUILD.bazel index d2c8419b81..6a0c167adc 100644 --- a/test/process_wrapper_bootstrap/BUILD.bazel +++ b/test/process_wrapper_bootstrap/BUILD.bazel @@ -1,12 +1,22 @@ -load("//rust:defs.bzl", "rust_test") -load(":process_wrapper_bootstrap_test.bzl", "process_wrapper_bootstrap_test_suite") +load("//rust:defs.bzl", "rust_binary", "rust_test") + +rust_binary( + name = "bootstrap_process_wrapper_probe", + srcs = ["bootstrap_process_wrapper_probe.rs"], + edition = "2021", +) rust_test( name = "bootstrap_process_wrapper_test", srcs = ["bootstrap_process_wrapper_test.rs"], - data = ["//util/process_wrapper/private:process_wrapper.sh"], + data = [ + ":bootstrap_process_wrapper_probe", + "//util/process_wrapper:bootstrap_process_wrapper", + ], edition = "2021", + env = { + "BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH": "$(rlocationpath :bootstrap_process_wrapper_probe)", + "BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH": "$(rlocationpath //util/process_wrapper:bootstrap_process_wrapper)", + }, deps = ["//rust/runfiles"], ) - -process_wrapper_bootstrap_test_suite(name = "process_wrapper_bootstrap_test_suite") diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs new file mode 100644 index 0000000000..35ecc496cf --- /dev/null +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs @@ -0,0 +1,10 @@ +fn main() { + let arg = std::env::args().nth(1).unwrap_or_default(); + println!("{arg}"); + + let exit_code = std::env::var("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + std::process::exit(exit_code); +} diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs index 4352d8d5de..43935b1b6e 100644 --- a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs @@ -1,24 +1,54 @@ -//! Tests for the bootstrap process wrapper +//! Tests for the bootstrap process wrapper. -use std::fs::read_to_string; +use std::env; +use std::process::Command; use runfiles::Runfiles; -/// Test that the shell process wrapper starts with the expected shebang to -/// avoid breaking the contract with the `bootstrap_process_wrapper` rule. -#[test] -fn test_shebang() { +fn resolve_runfile(env_var: &str) -> String { let rfiles = Runfiles::create().unwrap(); + let rlocationpath = env::var(env_var).unwrap(); + runfiles::rlocation!(rfiles, rlocationpath.as_str()) + .unwrap() + .display() + .to_string() +} - let script = runfiles::rlocation!( - rfiles, - "rules_rust/util/process_wrapper/private/process_wrapper.sh" - ) - .unwrap(); +#[test] +fn test_substitutes_pwd() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + let pwd = env::current_dir().unwrap().display().to_string(); + + let output = Command::new(wrapper) + .arg("--") + .arg(probe) + .arg("${pwd}/suffix") + .output() + .unwrap(); - let content = read_to_string(script).unwrap(); assert!( - content.starts_with("#!/bin/sh"), - "The shell script does not start with the expected shebang." - ) + output.status.success(), + "wrapper failed: status={:?}, stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr), + ); + + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.trim_end(), format!("{}/suffix", pwd)); +} + +#[test] +fn test_propagates_exit_code() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + + let status = Command::new(wrapper) + .arg("--") + .arg(probe) + .env("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE", "23") + .status() + .unwrap(); + + assert_eq!(status.code(), Some(23)); } diff --git a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl b/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl deleted file mode 100644 index 7e0e4ea571..0000000000 --- a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl +++ /dev/null @@ -1,78 +0,0 @@ -"""Starlark unit tests for the bootstrap process wrapper""" - -load("@bazel_skylib//lib:unittest.bzl", "analysistest") -load("//test/unit:common.bzl", "assert_action_mnemonic") - -def _enable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - if ctx.attr.expected_ext == ".bat": - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - else: - assert_action_mnemonic(env, target.actions[0], "TemplateExpand") - - return analysistest.end(env) - -_enable_sh_toolchain_test = analysistest.make( - _enable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): True, - }, - attrs = { - "expected_ext": attr.string( - doc = "The expected extension for the bootstrap script.", - mandatory = True, - values = [ - ".bat", - ".sh", - ], - ), - }, -) - -def _disable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - - return analysistest.end(env) - -_disable_sh_toolchain_test = analysistest.make( - _disable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): False, - }, -) - -def process_wrapper_bootstrap_test_suite(name, **kwargs): - """Entry-point macro called from the BUILD file. - - Args: - name (str): Name of the macro. - **kwargs (dict): Additional keyword arguments. - """ - - _enable_sh_toolchain_test( - name = "enable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - expected_ext = select({ - "@platforms//os:windows": ".bat", - "//conditions:default": ".sh", - }), - ) - - _disable_sh_toolchain_test( - name = "disable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - ) - - native.test_suite( - name = name, - tests = [ - ":disable_sh_toolchain_test", - ":enable_sh_toolchain_test", - ], - **kwargs - ) diff --git a/util/process_wrapper/BUILD.bazel b/util/process_wrapper/BUILD.bazel index d8ea9e02c9..dd8bd2992d 100644 --- a/util/process_wrapper/BUILD.bazel +++ b/util/process_wrapper/BUILD.bazel @@ -1,6 +1,5 @@ # buildifier: disable=bzl-visibility load("//rust/private:rust.bzl", "rust_binary_without_process_wrapper", "rust_test_without_process_wrapper_test") -load("//util/process_wrapper/private:bootstrap_process_wrapper.bzl", "bootstrap_process_wrapper") rust_binary_without_process_wrapper( name = "process_wrapper", @@ -33,11 +32,8 @@ rust_test_without_process_wrapper_test( edition = "2018", ) -bootstrap_process_wrapper( +alias( name = "bootstrap_process_wrapper", - is_windows = select({ - "@platforms//os:windows": True, - "//conditions:default": False, - }), + actual = "//util/process_wrapper/private:bootstrap_process_wrapper", visibility = ["//visibility:public"], ) diff --git a/util/process_wrapper/private/BUILD.bazel b/util/process_wrapper/private/BUILD.bazel index badd4a695d..6cbbbc07da 100644 --- a/util/process_wrapper/private/BUILD.bazel +++ b/util/process_wrapper/private/BUILD.bazel @@ -1,4 +1,7 @@ -exports_files([ - "process_wrapper.sh", - "process_wrapper.bat", -]) +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + +cc_binary( + name = "bootstrap_process_wrapper", + srcs = ["bootstrap_process_wrapper.cc"], + visibility = ["//util/process_wrapper:__pkg__"], +) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.bzl b/util/process_wrapper/private/bootstrap_process_wrapper.bzl deleted file mode 100644 index ca5047079c..0000000000 --- a/util/process_wrapper/private/bootstrap_process_wrapper.bzl +++ /dev/null @@ -1,73 +0,0 @@ -"""Bootstrap rustc process wrapper""" - -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - -def _bootstrap_process_wrapper_impl_unix(ctx): - output = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - setting = ctx.attr._use_sh_toolchain_for_bootstrap_process_wrapper[BuildSettingInfo].value - sh_toolchain = ctx.toolchains["@bazel_tools//tools/sh:toolchain_type"] - if setting and sh_toolchain: - shebang = "#!{}".format(sh_toolchain.path) - ctx.actions.expand_template( - output = output, - template = ctx.file._bash, - substitutions = { - # Replace the shebang with one constructed from the configured - # shell toolchain. - "#!/bin/sh": shebang, - }, - ) - else: - ctx.actions.symlink( - output = output, - target_file = ctx.file._bash, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl_windows(ctx): - output = ctx.actions.declare_file("{}.bat".format(ctx.label.name)) - ctx.actions.symlink( - output = output, - target_file = ctx.file._batch, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl(ctx): - if ctx.attr.is_windows: - return _bootstrap_process_wrapper_impl_windows(ctx) - return _bootstrap_process_wrapper_impl_unix(ctx) - -bootstrap_process_wrapper = rule( - doc = "A rule which produces a bootstrapping script for the rustc process wrapper.", - implementation = _bootstrap_process_wrapper_impl, - attrs = { - "is_windows": attr.bool( - doc = "Indicate whether or not the target platform is windows.", - mandatory = True, - ), - "_bash": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.sh"), - ), - "_batch": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.bat"), - ), - "_use_sh_toolchain_for_bootstrap_process_wrapper": attr.label( - default = Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper"), - ), - }, - toolchains = [config_common.toolchain_type("@bazel_tools//tools/sh:toolchain_type", mandatory = False)], - executable = True, -) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.cc b/util/process_wrapper/private/bootstrap_process_wrapper.cc new file mode 100644 index 0000000000..9b133da37e --- /dev/null +++ b/util/process_wrapper/private/bootstrap_process_wrapper.cc @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#include +#define getcwd _getcwd +#define stat _stat +#else +#include +#include +#endif + +namespace { + +constexpr const char* kPwdPlaceholder = "${pwd}"; +constexpr const char* kOutputBasePlaceholder = "${output_base}"; +constexpr const char* kExecRootPlaceholder = "${exec_root}"; + +#if defined(_WIN32) +constexpr char kPathSeparator = '\\'; +#else +constexpr char kPathSeparator = '/'; +#endif + +bool is_directory(const std::string& path) { + struct stat stat_buffer; + return stat(path.c_str(), &stat_buffer) == 0 && + (stat_buffer.st_mode & S_IFDIR) != 0; +} + +std::string dirname(const std::string& path) { + std::string::size_type slash = path.find_last_of("/\\"); + if (slash == std::string::npos) { + return path; + } + if (slash == 0) { + return path.substr(0, 1); + } + return path.substr(0, slash); +} + +std::string basename(const std::string& path) { + std::string::size_type slash = path.find_last_of("/\\"); + if (slash == std::string::npos) { + return path; + } + return path.substr(slash + 1); +} + +std::string join_path(const std::string& left, const std::string& right) { + if (left.empty()) { + return right; + } + if (left.back() == '/' || left.back() == '\\') { + return left + right; + } + return left + kPathSeparator + right; +} + +std::string canonicalize(const std::string& path) { +#if defined(_WIN32) + char* resolved = _fullpath(nullptr, path.c_str(), 0); +#else + char* resolved = realpath(path.c_str(), nullptr); +#endif + if (resolved == nullptr) { + return path; + } + std::string out = resolved; + std::free(resolved); + return out; +} + +std::string replace_all(std::string out, + const std::string& placeholder, + const std::string& replacement) { + std::string::size_type pos = 0; + while ((pos = out.find(placeholder, pos)) != std::string::npos) { + out.replace(pos, placeholder.size(), replacement); + pos += replacement.size(); + } + return out; +} + +std::string replace_placeholders(const std::string& arg, + const std::string& pwd, + const std::string& output_base, + const std::string& exec_root) { + std::string out = arg; + out = replace_all(out, kPwdPlaceholder, pwd); + out = replace_all(out, kOutputBasePlaceholder, output_base); + out = replace_all(out, kExecRootPlaceholder, exec_root); + return out; +} + +std::string get_output_base(const std::string& pwd) { + const std::string external = join_path(pwd, "external"); + if (is_directory(external)) { + return canonicalize(join_path(external, "..")); + } + return dirname(dirname(canonicalize(pwd))); +} + +std::vector build_exec_argv(const std::vector& args) { + std::vector exec_argv; + exec_argv.reserve(args.size() + 1); + for (const std::string& arg : args) { + exec_argv.push_back(const_cast(arg.c_str())); + } + exec_argv.push_back(nullptr); + return exec_argv; +} + +} // namespace + +int main(int argc, char** argv) { + int first_arg_index = 1; + if (argc > 1 && std::strcmp(argv[1], "--") == 0) { + first_arg_index = 2; + } + + if (first_arg_index >= argc) { + std::fprintf(stderr, "bootstrap_process_wrapper: missing command\n"); + return 1; + } + + char* pwd_raw = getcwd(nullptr, 0); + if (pwd_raw == nullptr) { + std::perror("bootstrap_process_wrapper: getcwd"); + return 1; + } + std::string pwd = pwd_raw; + std::free(pwd_raw); + const std::string output_base = get_output_base(pwd); + const std::string exec_root = + join_path(join_path(output_base, "execroot"), basename(pwd)); + + std::vector command_args; + command_args.reserve(static_cast(argc - first_arg_index)); + for (int i = first_arg_index; i < argc; ++i) { + command_args.push_back( + replace_placeholders(argv[i], pwd, output_base, exec_root)); + } + +#if defined(_WIN32) + for (char& c : command_args[0]) { + if (c == '/') { + c = '\\'; + } + } +#endif + + std::vector exec_argv = build_exec_argv(command_args); + +#if defined(_WIN32) + int exit_code = _spawnvp(_P_WAIT, exec_argv[0], exec_argv.data()); + if (exit_code == -1) { + std::perror("bootstrap_process_wrapper: _spawnvp"); + return 1; + } + return exit_code; +#else + execvp(exec_argv[0], exec_argv.data()); + std::perror("bootstrap_process_wrapper: execvp"); + return 1; +#endif +} diff --git a/util/process_wrapper/private/process_wrapper.bat b/util/process_wrapper/private/process_wrapper.bat deleted file mode 100755 index 70e9067fb7..0000000000 --- a/util/process_wrapper/private/process_wrapper.bat +++ /dev/null @@ -1,40 +0,0 @@ -@ECHO OFF -SETLOCAL enabledelayedexpansion - -SET command=%* - -:: Resolve the `${pwd}` placeholders -SET command=!command:${pwd}=%CD%! - -:: Resolve the `${output_base}` and `${exec_root}` placeholders. -:: The external directory is a junction/symlink to output_base\external. -:: This mirrors the logic in options.rs used by the real process wrapper. -FOR /F "delims=" %%i IN ('cd external\.. ^& cd') DO SET output_base=%%i -FOR %%i IN ("%CD%") DO SET workspace_name=%%~nxi -SET exec_root=!output_base!\execroot\!workspace_name! -SET command=!command:${output_base}=%output_base%! -SET command=!command:${exec_root}=%exec_root%! - -:: Strip out the leading `--` argument. -SET command=!command:~3! - -:: Find the rustc.exe argument and sanitize it's path -for %%A in (%*) do ( - SET arg=%%~A - if "!arg:~-9!"=="rustc.exe" ( - SET sanitized=!arg:/=\! - - SET command=!sanitized! !command:%%~A=! - goto :break - ) -) - -:break - -%command% - -:: Capture the exit code of rustc.exe -SET exit_code=!errorlevel! - -:: Exit with the same exit code -EXIT /b %exit_code% diff --git a/util/process_wrapper/private/process_wrapper.sh b/util/process_wrapper/private/process_wrapper.sh deleted file mode 100755 index f0ea720b05..0000000000 --- a/util/process_wrapper/private/process_wrapper.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -set -eu - -# Skip the first argument which is expected to be `--` -shift - -# Derive output_base and exec_root so we can expand their placeholders -# in rustc flags (e.g. --remap-path-prefix, -oso_prefix). This mirrors -# the logic in options.rs used by the real process wrapper. -phys_pwd=$(cd -P . && pwd) -if [ -d "external" ]; then - output_base=$(cd -P external/.. && pwd) -else - output_base="${phys_pwd%/*}" - output_base="${output_base%/*}" -fi -workspace_name="${phys_pwd##*/}" -exec_root="${output_base}/execroot/${workspace_name}" - -for arg in "$@"; do - case "$arg" in - *'${pwd}'*) - prefix="${arg%%\$\{pwd\}*}" - suffix="${arg#*\$\{pwd\}}" - arg="${prefix}${phys_pwd}${suffix}" - ;; - esac - case "$arg" in - *'${output_base}'*) - prefix="${arg%%\$\{output_base\}*}" - suffix="${arg#*\$\{output_base\}}" - arg="${prefix}${output_base}${suffix}" - ;; - esac - case "$arg" in - *'${exec_root}'*) - prefix="${arg%%\$\{exec_root\}*}" - suffix="${arg#*\$\{exec_root\}}" - arg="${prefix}${exec_root}${suffix}" - ;; - esac - set -- "$@" "$arg" - shift -done - -exec "$@" From c1d2ea9a10d4d7d876235ddc651a6bb3eb9bc072 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 18 Feb 2026 09:09:52 -0500 Subject: [PATCH 08/50] Attempt to fix CopyFile for windows --- MODULE.bazel | 1 + cargo/private/BUILD.bazel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 02d34b8c3a..7ba52073b9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,7 @@ module( ## Core ############################################################################### +bazel_dep(name = "bazel_lib", version = "3.0.0") bazel_dep(name = "bazel_features", version = "1.32.0") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "platforms", version = "1.1.0") diff --git a/cargo/private/BUILD.bazel b/cargo/private/BUILD.bazel index fd60c4ff62..536490441d 100644 --- a/cargo/private/BUILD.bazel +++ b/cargo/private/BUILD.bazel @@ -1,5 +1,5 @@ +load("@bazel_lib//lib:copy_file.bzl", "copy_file") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("//rust:defs.bzl", "rust_binary") rust_binary( From 320cb33f1e93ff551ed7083838458383d171c1bb Mon Sep 17 00:00:00 2001 From: isaacparker0 <128327439+isaacparker0@users.noreply.github.com> Date: Fri, 20 Feb 2026 09:19:32 -0500 Subject: [PATCH 09/50] Apply lint config in exec configuration (#2) --- rust/private/rustc.bzl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index c0fa573683..d8930253d3 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1544,11 +1544,8 @@ def rustc_compile_action( stamp = is_stamping_enabled(attr) # Add flags for any 'rustc' lints that are specified. - # - # Exclude lints if we're building in the exec configuration to prevent crates - # used in build scripts from generating warnings. lint_files = [] - if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config and not is_exec_configuration(ctx): + if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config: rust_flags = rust_flags + ctx.attr.lint_config[LintsInfo].rustc_lint_flags lint_files = lint_files + ctx.attr.lint_config[LintsInfo].rustc_lint_files From 5ed548f7c0bdd5452cb16a92a31752023af8e6fd Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 20 Feb 2026 09:43:40 -0500 Subject: [PATCH 10/50] Fix up rules_rust bzl_library targets --- cargo/private/BUILD.bazel | 5 ++++- rust/platform/BUILD.bazel | 1 - rust/private/BUILD.bazel | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cargo/private/BUILD.bazel b/cargo/private/BUILD.bazel index 536490441d..db04afa7b1 100644 --- a/cargo/private/BUILD.bazel +++ b/cargo/private/BUILD.bazel @@ -39,6 +39,9 @@ copy_file( bzl_library( name = "bzl_lib", + deps = [ + "//rust:bzl_lib", + ], srcs = glob(["**/*.bzl"]), - visibility = ["//:__subpackages__"], + visibility = ["//visibility:public"], ) diff --git a/rust/platform/BUILD.bazel b/rust/platform/BUILD.bazel index d0246e6ae6..fcbc677b6f 100644 --- a/rust/platform/BUILD.bazel +++ b/rust/platform/BUILD.bazel @@ -33,5 +33,4 @@ package_group( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], ) diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index d18e895493..d3cea4650d 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -22,7 +22,7 @@ bzl_library( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], + visibility = ["//visibility:public"], deps = [ ":bazel_tools_bzl_lib", ":rules_cc_bzl_lib", From 08a7b7df74a026c8ae46b4b8b6aba008a1f35567 Mon Sep 17 00:00:00 2001 From: isaacparker0 <128327439+isaacparker0@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:32:15 -0500 Subject: [PATCH 11/50] rust-analyzer: include Bazel package dir in crate source include_dirs (#3) * 0 * Add rust analyzer test coverage --- rust/private/rust_analyzer.bzl | 9 +++ .../rust_project_json_test.rs | 32 ++++---- .../BUILD.bazel | 29 +++++++ .../subdir_test_crates_same_package/lib.rs | 1 + .../rust_project_json_test.rs | 76 +++++++++++++++++++ .../subdir/subdir_test.rs | 5 ++ 6 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/lib.rs create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs diff --git a/rust/private/rust_analyzer.bzl b/rust/private/rust_analyzer.bzl index 9ad74160d8..23761a5ce0 100644 --- a/rust/private/rust_analyzer.bzl +++ b/rust/private/rust_analyzer.bzl @@ -265,6 +265,15 @@ def _create_single_crate(ctx, attrs, info): crate["root_module"] = _WORKSPACE_TEMPLATE + src_map[info.crate.root.short_path].path crate["source"]["include_dirs"].append(path_prefix + info.crate.root.dirname) + # Ensure workspace crates in the same Bazel package share one source root. + # + # rust-analyzer picks candidate crates by source root (`relevant_crates`). + # Widening include_dirs at the package level keeps related crates in a + # shared candidate set; final membership is still resolved by each crate's + # module tree. + if not is_external: + crate["source"]["include_dirs"].append(_WORKSPACE_TEMPLATE + ctx.label.package) + if info.build_info != None and info.build_info.out_dir != None: out_dir_path = info.build_info.out_dir.path crate["env"].update({"OUT_DIR": _EXEC_ROOT_TEMPLATE + out_dir_path}) diff --git a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs index d9ce7579c5..2138c62e19 100644 --- a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs +++ b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs @@ -2,11 +2,12 @@ mod tests { use serde::Deserialize; use std::env; + use std::fs; + use std::path::Path; use std::path::PathBuf; #[derive(Deserialize)] struct Project { - sysroot_src: String, crates: Vec, } @@ -25,22 +26,12 @@ mod tests { #[test] fn test_generated_srcs() { let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let rust_project_path = fs::canonicalize(&rust_project_path).unwrap(); let content = std::fs::read_to_string(&rust_project_path) .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); let project: Project = serde_json::from_str(&content).expect("Failed to deserialize project JSON"); - // /tmp/_bazel/12345678/external/tools/rustlib/library => /tmp/_bazel - let output_base = project - .sysroot_src - .rsplitn(2, "/external/") - .last() - .unwrap() - .rsplitn(2, '/') - .last() - .unwrap(); - println!("output_base: {output_base}"); - let with_gen = project .crates .iter() @@ -50,7 +41,20 @@ mod tests { assert!(with_gen.root_module.ends_with("/lib.rs")); let include_dirs = &with_gen.source.as_ref().unwrap().include_dirs; - assert!(include_dirs.len() == 1); - assert!(include_dirs[0].starts_with(output_base)); + assert_eq!(include_dirs.len(), 2); + + let root_module_parent = Path::new(&with_gen.root_module).parent().unwrap(); + let workspace_dir = rust_project_path.parent().unwrap(); + + assert!( + include_dirs.iter().any(|p| Path::new(p) == root_module_parent), + "expected include_dirs to contain root_module parent, got include_dirs={include_dirs:?}, root_module={}", + with_gen.root_module, + ); + assert!( + include_dirs.iter().any(|p| Path::new(p) == workspace_dir), + "expected include_dirs to contain workspace dir, got include_dirs={include_dirs:?}, workspace_dir={}", + workspace_dir.display(), + ); } } diff --git a/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel new file mode 100644 index 0000000000..eac2fe6e4e --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "mylib", + srcs = ["lib.rs"], + edition = "2018", +) + +rust_test( + name = "mylib_test", + srcs = ["subdir/subdir_test.rs"], + edition = "2018", +) + +rust_test( + name = "rust_project_json_test", + srcs = ["rust_project_json_test.rs"], + data = [":rust-project.json"], + edition = "2018", + env = {"RUST_PROJECT_JSON": "$(rootpath :rust-project.json)"}, + # This target is tagged as manual since it's not expected to pass in + # contexts outside of `//test/rust_analyzer:rust_analyzer_test`. Run + # that target to execute this test. + tags = ["manual"], + deps = [ + "//test/rust_analyzer/3rdparty/crates:serde", + "//test/rust_analyzer/3rdparty/crates:serde_json", + ], +) diff --git a/test/rust_analyzer/subdir_test_crates_same_package/lib.rs b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs @@ -0,0 +1 @@ + diff --git a/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs new file mode 100644 index 0000000000..77a99d75ed --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs @@ -0,0 +1,76 @@ +#[cfg(test)] +mod tests { + use serde::Deserialize; + use std::collections::BTreeSet; + use std::env; + use std::path::PathBuf; + + #[derive(Deserialize)] + struct Project { + crates: Vec, + } + + #[derive(Deserialize)] + struct Crate { + root_module: String, + is_workspace_member: Option, + source: Option, + } + + #[derive(Deserialize)] + struct Source { + include_dirs: Vec, + } + + fn normalize(path: &str) -> String { + path.trim_end_matches('/').to_owned() + } + + #[test] + fn test_same_package_crates_share_include_dir() { + let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let content = std::fs::read_to_string(&rust_project_path) + .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); + let project: Project = + serde_json::from_str(&content).expect("Failed to deserialize project JSON"); + + let lib = project + .crates + .iter() + .find(|c| c.is_workspace_member == Some(true) && c.root_module.ends_with("/lib.rs")) + .expect("missing library crate"); + let test = project + .crates + .iter() + .find(|c| { + c.is_workspace_member == Some(true) + && c.root_module.ends_with("/subdir/subdir_test.rs") + }) + .expect("missing subdir test crate"); + + let lib_include_dirs: BTreeSet<_> = lib + .source + .as_ref() + .expect("lib crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + let test_include_dirs: BTreeSet<_> = test + .source + .as_ref() + .expect("test crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + + let shared_dir = lib_include_dirs + .intersection(&test_include_dirs) + .next() + .expect("expected crates in same package to share an include_dir"); + + assert!(lib.root_module.starts_with(&format!("{}/", shared_dir))); + assert!(test.root_module.starts_with(&format!("{}/", shared_dir))); + } +} diff --git a/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs new file mode 100644 index 0000000000..915d7c6130 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs @@ -0,0 +1,5 @@ +#[test] +fn test_subdir_fixture() { + let marker = String::from("ok"); + assert_eq!(marker.len(), 2); +} From dec54f70d884f1db1bbd4d96ca82046c43e1c14a Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 9 Sep 2025 09:19:46 -0400 Subject: [PATCH 12/50] Improve proc_macro_deps ergonomics --- .../private/wasm_bindgen_test.bzl | 7 +- rust/defs.bzl | 26 ++++-- rust/private/rust.bzl | 81 +++++++------------ rust/private/rustc.bzl | 2 +- rust/private/rustdoc/BUILD.bazel | 2 +- rust/private/rustdoc_test.bzl | 8 +- rust/private/utils.bzl | 28 ++++++- 7 files changed, 85 insertions(+), 69 deletions(-) diff --git a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl index 925a11a2f4..6c26f6c3cf 100644 --- a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl +++ b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl @@ -13,6 +13,7 @@ load( "@rules_rust//rust/private:utils.bzl", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_import_macro_deps", @@ -61,8 +62,10 @@ def _rust_wasm_bindgen_test_binary_impl(ctx): toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps + [wb_toolchain.wasm_bindgen_test]) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps + [wb_toolchain.wasm_bindgen_test]) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) # Target is building the crate in `test` config if WasmBindgenTestCrateInfo in ctx.attr.wasm: diff --git a/rust/defs.bzl b/rust/defs.bzl index 4f2ef72582..09be66d253 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -78,25 +78,37 @@ load( _rust_unpretty_aspect = "rust_unpretty_aspect", ) -rust_library = _rust_library +def _rule_wrapper(rule): + def _wrapped(name, deps = [], proc_macro_deps = [], **kwargs): + rule( + name = name, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return _wrapped + +rust_library = _rule_wrapper(_rust_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_static_library = _rust_static_library +rust_static_library = _rule_wrapper(_rust_static_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_shared_library = _rust_shared_library +rust_shared_library = _rule_wrapper(_rust_shared_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_proc_macro = _rust_proc_macro +rust_proc_macro = _rule_wrapper(_rust_proc_macro) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_binary = _rust_binary +rust_binary = _rule_wrapper(_rust_binary) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_library_group = _rust_library_group # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_test = _rust_test +rust_test = _rule_wrapper(_rust_test) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_test_suite = _rust_test_suite @@ -105,7 +117,7 @@ rust_test_suite = _rust_test_suite rust_doc = _rust_doc # See @rules_rust//rust/private:rustdoc.bzl for a complete description. -rust_doc_test = _rust_doc_test +rust_doc_test = _rule_wrapper(_rust_doc_test) # See @rules_rust//rust/private:rustdoc_test.bzl for a complete description. clippy_flag = _clippy_flag diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 301eec048e..01e1b709d4 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -23,6 +23,7 @@ load( "BuildInfo", "CrateGroupInfo", "CrateInfo", + "DepInfo", "LintsInfo", "UnstableRustFeaturesInfo", ) @@ -47,6 +48,7 @@ load( "determine_lib_name", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_edition", @@ -65,41 +67,6 @@ def _assert_no_deprecated_attributes(_ctx): """ pass -def _assert_correct_dep_mapping(ctx): - """Forces a failure if proc_macro_deps and deps are mixed inappropriately - - Args: - ctx (ctx): The current rule's context object - """ - for dep in ctx.attr.deps: - if rust_common.crate_info in dep: - if dep[rust_common.crate_info].type == "proc-macro": - fail( - "{} listed {} in its deps, but it is a proc-macro. It should instead be in the bazel property proc_macro_deps.".format( - ctx.label, - dep.label, - ), - ) - for dep in ctx.attr.proc_macro_deps: - if CrateInfo in dep: - types = [dep[CrateInfo].type] - else: - types = [ - dep_variant_info.crate_info.type - for dep_variant_info in dep[CrateGroupInfo].dep_variant_infos.to_list() - if dep_variant_info.crate_info - ] - - for type in types: - if type != "proc-macro": - fail( - "{} listed {} in its proc_macro_deps, but it is not proc-macro, it is a {}. It should probably instead be listed in deps.".format( - ctx.label, - dep.label, - type, - ), - ) - def _rust_library_impl(ctx): """The implementation of the `rust_library` rule. @@ -169,7 +136,7 @@ def _rust_library_common(ctx, crate_type): list: A list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) @@ -216,8 +183,8 @@ def _rust_library_common(ctx, crate_type): not ctx.attr.disable_pipelining ) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) return rustc_compile_action( ctx = ctx, @@ -260,7 +227,7 @@ def _rust_binary_impl(ctx): """ toolchain = find_toolchain(ctx) crate_name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) if ctx.attr.binary_name: output_filename = ctx.attr.binary_name @@ -268,8 +235,8 @@ def _rust_binary_impl(ctx): output_filename = ctx.label.name output = ctx.actions.declare_file(output_filename + toolchain.binary_ext) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) crate_root = getattr(ctx.file, "crate_root", None) if not crate_root: @@ -350,13 +317,13 @@ def _rust_test_impl(ctx): list: The list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) if ctx.attr.crate and ctx.attr.srcs: fail("rust_test.crate and rust_test.srcs are mutually exclusive. Update {} to use only one of these attributes".format( @@ -547,16 +514,16 @@ def _rust_library_group_impl(ctx): runfiles = [] for dep in ctx.attr.deps: - if rust_common.crate_info in dep: + if CrateInfo in dep: dep_variant_infos.append(rust_common.dep_variant_info( - crate_info = dep[rust_common.crate_info] if rust_common.crate_info in dep else None, - dep_info = dep[rust_common.dep_info] if rust_common.crate_info in dep else None, + crate_info = dep[CrateInfo] if CrateInfo in dep else None, + dep_info = dep[DepInfo] if DepInfo in dep else None, build_info = dep[BuildInfo] if BuildInfo in dep else None, cc_info = dep[CcInfo] if CcInfo in dep else None, crate_group_info = None, )) - elif rust_common.crate_group_info in dep: - dep_variant_transitive_infos.append(dep[rust_common.crate_group_info].dep_variant_infos) + elif CrateGroupInfo in dep: + dep_variant_transitive_infos.append(dep[CrateGroupInfo].dep_variant_infos) else: fail("crate_group_info targets can only depend on rust_library or rust_library_group targets.") @@ -732,10 +699,12 @@ _COMMON_ATTRS = { # `@local_config_platform//:exec` exposed. "proc_macro_deps": attr.label_list( doc = dedent("""\ - List of `rust_proc_macro` targets used to help build this library target. + Copy of deps in exec configuration. This should really be called `exec_configured_deps`. + + Rule implementations use this to select exec-configured `rust_proc_macro` targets. + User code should pass all deps to `deps` for the macros loaded from `defs.bzl`. """), cfg = "exec", - providers = [[CrateInfo], [CrateGroupInfo]], ), "require_explicit_unstable_features": attr.int( doc = ( @@ -1355,7 +1324,9 @@ rust_binary_without_process_wrapper = rule( implementation = _rust_binary_without_process_wrapper_impl, doc = "A variant of `rust_binary` that uses a minimal process wrapper for `Rustc` actions.", provides = COMMON_PROVIDERS + [_RustBuiltWithoutProcessWrapperInfo], - attrs = _common_attrs_for_binary_without_process_wrapper(_COMMON_ATTRS | _RUST_BINARY_ATTRS), + attrs = _common_attrs_for_binary_without_process_wrapper(_COMMON_ATTRS | _RUST_BINARY_ATTRS | { + "_skip_deps_verification": attr.bool(default = True), + }), executable = True, fragments = ["cpp"], toolchains = [ @@ -1591,7 +1562,7 @@ rust_test = rule( """), ) -def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): +def rust_test_suite(name, srcs, shared_srcs = [], deps = [], proc_macro_deps = [], **kwargs): """A rule for creating a test suite for a set of `rust_test` targets. This rule can be used for setting up typical rust [integration tests][it]. Given the following @@ -1644,6 +1615,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): name (str): The name of the `test_suite`. srcs (list): All test sources, typically `glob(["tests/**/*.rs"])`. shared_srcs (list): Optional argument for sources shared among tests, typically helper functions. + deps (list): Deps and proc_macro_deps for underlying test. + proc_macro_deps (list): Deprecated; do not use. **kwargs (dict): Additional keyword arguments for the underlying [rust_test](#rust_test) targets. The `tags` argument is also passed to the generated `test_suite` target. """ @@ -1674,6 +1647,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): srcs = [src] + shared_srcs, tags = tags, crate_name = crate_name, + deps = deps + proc_macro_deps, + proc_macro_deps = deps + proc_macro_deps, **kwargs ) tests.append(test_name) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index d8930253d3..76c2d9ca67 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -226,7 +226,7 @@ def collect_deps( Args: deps (list): The deps from ctx.attr.deps. - proc_macro_deps (list): The proc_macro deps from ctx.attr.proc_macro_deps. + proc_macro_deps (list): The proc_macro deps from `filter_deps(ctx)`. aliases (dict): A dict mapping aliased targets to their actual Crate information. Returns: diff --git a/rust/private/rustdoc/BUILD.bazel b/rust/private/rustdoc/BUILD.bazel index ee2067a87d..85cda1cd7e 100644 --- a/rust/private/rustdoc/BUILD.bazel +++ b/rust/private/rustdoc/BUILD.bazel @@ -1,4 +1,4 @@ -load("//rust/private:rust.bzl", "rust_binary") +load("//rust:defs.bzl", "rust_binary") package(default_visibility = ["//visibility:public"]) diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index 522c048cd1..dcf847f10a 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -18,7 +18,7 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//rust/private:common.bzl", "rust_common") load("//rust/private:providers.bzl", "CrateInfo") load("//rust/private:rustdoc.bzl", "rustdoc_compile_action") -load("//rust/private:utils.bzl", "dedent", "find_toolchain", "transform_deps") +load("//rust/private:utils.bzl", "dedent", "filter_deps", "find_toolchain", "transform_deps") def _construct_writer_arguments(ctx, test_runner, opt_test_params, action, crate_info): """Construct arguments and environment variables specific to `rustdoc_test_writer`. @@ -110,8 +110,10 @@ def _rust_doc_test_impl(ctx): toolchain = find_toolchain(ctx) crate = ctx.attr.crate[rust_common.crate_info] - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps) crate_info = rust_common.create_crate_info( name = crate.name, diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index 9c2b4dd9b6..c84610b126 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -510,14 +510,38 @@ def is_exec_configuration(ctx): # TODO(djmarcin): Is there any better way to determine cfg=exec? return ctx.genfiles_dir.path.find("-exec") != -1 +def filter_deps(ctx): + """Filters the provided (combined) deps into normal deps and proc_macro deps. + + Args: + ctx (ctx): The current rule's context object + + Returns: + deps and proc_macro_deps + """ + if len(ctx.attr.deps) != len(ctx.attr.proc_macro_deps) and not getattr(ctx.attr, "_skip_deps_verification", False): + fail("All deps should be passed to both `deps` and `proc_macro_deps`; please use the macros in //rust:defs.bzl") + + deps = [] + for dep in ctx.attr.deps: + if CrateInfo not in dep or dep[CrateInfo].type != "proc-macro": + deps.append(dep) + + proc_macro_deps = [] + for dep in ctx.attr.proc_macro_deps: + if CrateInfo in dep and dep[CrateInfo].type == "proc-macro": + proc_macro_deps.append(dep) + + return deps, proc_macro_deps + def transform_deps(deps): """Transforms a [Target] into [DepVariantInfo]. - This helper function is used to transform ctx.attr.deps and ctx.attr.proc_macro_deps into + This helper function is used to transform deps and .proc_macro_deps coming from `filter_deps` into [DepVariantInfo]. Args: - deps (list of Targets): Dependencies coming from ctx.attr.deps or ctx.attr.proc_macro_deps + deps (list of Targets): Dependencies coming from `filter_deps` Returns: list of DepVariantInfos. From 4d74eef11330bbd5cd93690cd8e5373d1a9c642e Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 25 Feb 2026 04:15:34 -0500 Subject: [PATCH 13/50] Always use param file for process wrapper --- rust/private/rustc.bzl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 76c2d9ca67..4bf3608fd4 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -956,6 +956,7 @@ def construct_arguments( force_depend_on_objects = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, + always_use_param_file = False, error_format = None, allowed_unstable_rust_features = None): """Builds an Args object containing common rustc flags @@ -1125,7 +1126,7 @@ def construct_arguments( # Rustc arguments rustc_flags = ctx.actions.args() rustc_flags.set_param_file_format("multiline") - rustc_flags.use_param_file("@%s", use_always = False) + rustc_flags.use_param_file("@%s", use_always = always_use_param_file) rustc_flags.add(crate_info.root) rustc_flags.add(crate_info.name, format = "--crate-name=%s") rustc_flags.add(crate_info.type, format = "--crate-type=%s") @@ -1612,6 +1613,7 @@ def rustc_compile_action( use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, + always_use_param_file = not ctx.executable._process_wrapper, allowed_unstable_rust_features = allowed_unstable_rust_features, ) From a37d122aa3004ff4b02bda4a480caa6b4d6704a6 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 26 Feb 2026 10:49:57 -0500 Subject: [PATCH 14/50] Avoid hashing RustAnalyzerInfo in rust_analyzer alias mapping --- rust/private/rust_analyzer.bzl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/private/rust_analyzer.bzl b/rust/private/rust_analyzer.bzl index 23761a5ce0..89b8dd1f0d 100644 --- a/rust/private/rust_analyzer.bzl +++ b/rust/private/rust_analyzer.bzl @@ -132,10 +132,13 @@ def _rust_analyzer_aspect_impl(target, ctx): else: fail("Unexpected target type: {}".format(target)) - aliases = {} + # Keep aliases as a list of (RustAnalyzerInfo, alias_name) tuples. + # Using RustAnalyzerInfo as a dict key can trigger expensive recursive hashing. + aliases = [] for aliased_target, aliased_name in getattr(ctx.rule.attr, "aliases", {}).items(): - if aliased_target.label in labels_to_rais: - aliases[labels_to_rais[aliased_target.label]] = aliased_name + dep_info = labels_to_rais.get(aliased_target.label) + if dep_info: + aliases.append((dep_info, aliased_name)) proc_macro_dylib = find_proc_macro_dylib(toolchain, target) proc_macro_dylibs = [proc_macro_dylib] if proc_macro_dylib else None @@ -301,7 +304,7 @@ def _create_single_crate(ctx, attrs, info): # the crate being processed, we don't add it as a dependency to itself. This is # common and expected - `rust_test.crate` pointing to the `rust_library`. crate["deps"] = [_crate_id(dep.crate) for dep in info.deps if _crate_id(dep.crate) != crate_id] - crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases.items()} + crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases} crate["cfg"] = info.cfgs toolchain = find_toolchain(ctx) crate["target"] = (_EXEC_ROOT_TEMPLATE + toolchain.target_json.path) if toolchain.target_json else toolchain.target_flag_value From be96cef2423fbf587db6f662c1e45cdbe0f97e88 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 26 Feb 2026 11:33:43 -0500 Subject: [PATCH 15/50] Convert wrappers to symbolic macros --- rust/defs.bzl | 27 ++++++++++++++++++++++++--- rust/rust_binary.bzl | 2 +- rust/rust_library.bzl | 2 +- rust/rust_test.bzl | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/rust/defs.bzl b/rust/defs.bzl index 09be66d253..8c6c14813d 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -14,6 +14,7 @@ """Public entry point to all Rust rules and supported APIs.""" +load("@bazel_features//:features.bzl", "bazel_features") load( "//rust:toolchain.bzl", _rust_stdlib_filegroup = "rust_stdlib_filegroup", @@ -90,7 +91,27 @@ def _rule_wrapper(rule): return _wrapped -rust_library = _rule_wrapper(_rust_library) +def _symbolic_rule_wrapper(rule, macro_fn): + def _wrapped(name, visibility, deps, proc_macro_deps, **kwargs): + rule( + name = name, + visibility = visibility, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return macro_fn( + implementation = _wrapped, + inherit_attrs = rule, + attrs = { + "deps": attr.label_list(default = []), + "proc_macro_deps": attr.label_list(default = []), + }, + ) + +rust_library = _symbolic_rule_wrapper(_rust_library, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_library) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_static_library = _rule_wrapper(_rust_static_library) @@ -102,13 +123,13 @@ rust_shared_library = _rule_wrapper(_rust_shared_library) rust_proc_macro = _rule_wrapper(_rust_proc_macro) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_binary = _rule_wrapper(_rust_binary) +rust_binary = _symbolic_rule_wrapper(_rust_binary, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_binary) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_library_group = _rust_library_group # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_test = _rule_wrapper(_rust_test) +rust_test = _symbolic_rule_wrapper(_rust_test, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_test) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_test_suite = _rust_test_suite diff --git a/rust/rust_binary.bzl b/rust/rust_binary.bzl index dbeba03911..518c7c3f01 100644 --- a/rust/rust_binary.bzl +++ b/rust/rust_binary.bzl @@ -1,7 +1,7 @@ """rust_binary""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_binary = "rust_binary", ) diff --git a/rust/rust_library.bzl b/rust/rust_library.bzl index b1e63494a6..a694ed970c 100644 --- a/rust/rust_library.bzl +++ b/rust/rust_library.bzl @@ -1,7 +1,7 @@ """rust_library""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_library = "rust_library", ) diff --git a/rust/rust_test.bzl b/rust/rust_test.bzl index 001963fccc..abd61ba8c5 100644 --- a/rust/rust_test.bzl +++ b/rust/rust_test.bzl @@ -1,7 +1,7 @@ """rust_test""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_test = "rust_test", ) From 431830ab0c7471e1a601f8626f56ce5b80fdc75e Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 28 Feb 2026 12:52:38 -0500 Subject: [PATCH 16/50] Handle toolchain registration when not registered as a bazel_dep --- rust/extensions.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/extensions.bzl b/rust/extensions.bzl index 2ce6a5226a..a8a1ce0bb5 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -32,8 +32,6 @@ def _find_modules(module_ctx): our_module = mod if root == None: root = our_module - if our_module == None: - fail("Unable to find rules_rust module") return root, our_module @@ -110,7 +108,9 @@ def _rust_impl(module_ctx): if toolchain_triples.get(repository_set["exec_triple"]) == repository_set["name"]: toolchain_triples.pop(repository_set["exec_triple"], None) - toolchains = root.tags.toolchain or rules_rust.tags.toolchain + toolchains = root.tags.toolchain + if not toolchains and rules_rust: + toolchains = rules_rust.tags.toolchain for toolchain in toolchains: if toolchain.extra_rustc_flags and toolchain.extra_rustc_flags_triples: From 189ca8c08143b7fd53e224a924fb3cce154bcef9 Mon Sep 17 00:00:00 2001 From: Walter Gray Date: Wed, 25 Feb 2026 18:32:17 -0800 Subject: [PATCH 17/50] Replace the --rustc-quit-on-rmeta / .rmeta approach with Buck2-style hollow rlibs: the RustcMetadata action runs rustc to completion with -Zno-codegen, emitting a .rlib archive. This approach mirrors the one used by buck2 and avoids needing to kill rustc mid-output in order to produce metadata. While not fixing problems with SVH mismatches when non-determinism, this does simplify the codepath and uses a production tested technique that doesn't have any of the dangers associated with killing the rustc process while it's still active. --- extensions/prost/private/prost.bzl | 4 +- rust/private/clippy.bzl | 2 +- rust/private/rust.bzl | 8 +- rust/private/rustc.bzl | 37 +++--- rust/private/unpretty.bzl | 2 +- rust/settings/settings.bzl | 11 +- test/process_wrapper/BUILD.bazel | 4 +- ...uit_on_rmeta.rs => rustc_output_format.rs} | 54 +++------ .../metadata_output_groups.bzl | 4 +- .../pipelined_compilation_test.bzl | 112 ++++++++++-------- test/unit/pipelined_compilation/wrap.bzl | 2 +- .../codegen_disambiguation_test.bzl | 2 +- util/process_wrapper/main.rs | 43 ------- 13 files changed, 127 insertions(+), 158 deletions(-) rename test/process_wrapper/{rustc_quit_on_rmeta.rs => rustc_output_format.rs} (63%) diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index 4eecf85a06..6a8d36e10d 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -180,7 +180,7 @@ def _compile_rust( prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) lib = ctx.actions.declare_file(lib_name) @@ -193,7 +193,7 @@ def _compile_rust( prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) rmeta = ctx.actions.declare_file(rmeta_name) rustc_rmeta_output = generate_output_diagnostics(ctx, rmeta) diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index 2b42e18e4e..3f54e4162e 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -177,7 +177,7 @@ def rust_clippy_action(ctx, clippy_executable, process_wrapper, crate_info, conf out_dir = out_dir, build_env_files = build_env_files, build_flags_files = build_flags_files, - emit = ["dep-info", "metadata"], + emit = ["metadata"], skip_expanding_rustc_env = True, use_json_output = bool(clippy_diagnostics_file), error_format = error_format, diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 01e1b709d4..496cabb000 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -174,7 +174,7 @@ def _rust_library_common(ctx, crate_type): disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): rust_metadata = ctx.actions.declare_file( - paths.replace_extension(rust_lib_name, ".rmeta"), + paths.replace_extension(rust_lib_name, "_meta.rlib"), sibling = rust_lib, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -247,7 +247,7 @@ def _rust_binary_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, ctx.attr.crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -349,7 +349,7 @@ def _rust_test_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -419,7 +419,7 @@ def _rust_test_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 4bf3608fd4..bcfce3e981 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -945,7 +945,7 @@ def construct_arguments( build_flags_files, tool_file = None, tool_path = None, - emit = ["dep-info", "link"], + emit = ["link"], force_all_deps_direct = False, add_flags_for_binary = False, include_link_flags = True, @@ -1154,8 +1154,11 @@ def construct_arguments( error_format = "json" if build_metadata: - # Configure process_wrapper to terminate rustc when metadata are emitted - process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") + # Build a hollow rlib (metadata-full, Buck2 equivalent) using -Zno-codegen. + # This produces an rlib with metadata but no object code, allowing downstream + # crates to start compiling without waiting for codegen. + # RUSTC_BOOTSTRAP=1 must be set in the action env for this unstable flag. + rustc_flags.add("-Zno-codegen") if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output) elif crate_info.rustc_output: @@ -1197,7 +1200,12 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and build_metadata and crate_info.metadata != None: + # Redirect hollow rlib output to the declared metadata file path, + # since -Zno-codegen --emit=link would otherwise write lib.rlib + # which collides with the full action's output. + rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") + elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1566,17 +1574,12 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) + compile_inputs_metadata = compile_inputs + # The types of rustc outputs to emit. - # If we build metadata, we need to keep the command line of the two invocations - # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as - # a candidate. - # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. - # # When cc_common linking is enabled, emit a `.o` file, which is later # passed to the cc_common.link action. - emit = ["dep-info", "link"] - if build_metadata: - emit.append("metadata") + emit = ["link"] if experimental_use_cc_common_link: emit = ["obj"] @@ -1626,7 +1629,7 @@ def rustc_compile_action( toolchain = toolchain, tool_file = toolchain.rustc, cc_toolchain = cc_toolchain, - emit = emit, + emit = ["link"], feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1650,6 +1653,12 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) + if build_metadata: + # RUSTC_BOOTSTRAP=1 is required for -Zno-codegen on stable rustc, and must + # be set on both the metadata and full actions for SVH compatibility (since + # RUSTC_BOOTSTRAP affects the crate hash). + env["RUSTC_BOOTSTRAP"] = "1" + if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1722,7 +1731,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs, + inputs = compile_inputs_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, diff --git a/rust/private/unpretty.bzl b/rust/private/unpretty.bzl index 0071d81943..b8ba474fd5 100644 --- a/rust/private/unpretty.bzl +++ b/rust/private/unpretty.bzl @@ -203,7 +203,7 @@ def _rust_unpretty_aspect_impl(target, ctx): out_dir = out_dir, build_env_files = build_env_files, build_flags_files = build_flags_files, - emit = ["dep-info", "metadata"], + emit = ["metadata"], skip_expanding_rustc_env = True, ) diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index b6d1c0aea4..bb232bebe2 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,10 +112,15 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. + """When set, this flag enables pipelined compilation for `rlib -> rlib` dependencies. - While this involves one extra (short) rustc invocation to build the rmeta file, - it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. + For each library target, a second RustcMetadata action is created that runs rustc with + `-Zno-codegen --emit=link` to produce a hollow rlib (metadata & MIR, no object code). + Downstream library compilations can start as soon as this hollow rlib is available, + increasing parallelism during compilation. + + Requires RUSTC_BOOTSTRAP=1, which is set automatically on both the metadata and full + actions for pipelined targets. """ bool_flag( name = "pipelined_compilation", diff --git a/test/process_wrapper/BUILD.bazel b/test/process_wrapper/BUILD.bazel index 43e2420062..1ab6510841 100644 --- a/test/process_wrapper/BUILD.bazel +++ b/test/process_wrapper/BUILD.bazel @@ -157,8 +157,8 @@ rust_binary( ) rust_test( - name = "rustc_quit_on_rmeta", - srcs = ["rustc_quit_on_rmeta.rs"], + name = "rustc_output_format", + srcs = ["rustc_output_format.rs"], data = [ ":fake_rustc", "//util/process_wrapper", diff --git a/test/process_wrapper/rustc_quit_on_rmeta.rs b/test/process_wrapper/rustc_output_format.rs similarity index 63% rename from test/process_wrapper/rustc_quit_on_rmeta.rs rename to test/process_wrapper/rustc_output_format.rs index b804ba1ebb..a3b175a48a 100644 --- a/test/process_wrapper/rustc_quit_on_rmeta.rs +++ b/test/process_wrapper/rustc_output_format.rs @@ -40,54 +40,38 @@ mod test { } #[test] - fn test_rustc_quit_on_rmeta_quits() { - let out_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "rendered", - ], - &[], - true, + fn test_rustc_output_format_rendered() { + let out_content = fake_rustc(&["--rustc-output-format", "rendered"], &[], true); + assert!( + out_content.contains("should be\nin output"), + "output should contain the first rendered message", ); assert!( - !out_content.contains("should not be in output"), - "output should not contain 'should not be in output' but did", + out_content.contains("should not be in output"), + "output should contain the second rendered message", + ); + assert!( + !out_content.contains(r#""rendered""#), + "rendered mode should not print raw json", ); } #[test] - fn test_rustc_quit_on_rmeta_output_json() { + fn test_rustc_output_format_json() { let json_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "json", - ], + &["--rustc-output-format", "json"], &[], true, ); assert_eq!( json_content, - concat!(r#"{"rendered": "should be\nin output"}"#, "\n") - ); - } - - #[test] - fn test_rustc_quit_on_rmeta_output_rendered() { - let rendered_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "rendered", - ], - &[], - true, + concat!( + r#"{"rendered": "should be\nin output"}"#, + "\n", + r#"{"rendered": "should not be in output"}"#, + "\n" + ) ); - assert_eq!(rendered_content, "should be\nin output"); } #[test] diff --git a/test/unit/metadata_output_groups/metadata_output_groups.bzl b/test/unit/metadata_output_groups/metadata_output_groups.bzl index ec89e0f132..a20c9ac5b2 100644 --- a/test/unit/metadata_output_groups/metadata_output_groups.bzl +++ b/test/unit/metadata_output_groups/metadata_output_groups.bzl @@ -13,8 +13,8 @@ def _metadata_output_groups_present_test_impl(ctx): asserts.equals(env, 1, len(build_metadata), "Expected 1 build_metadata file") asserts.true( - build_metadata[0].basename.endswith(".rmeta"), - "Expected %s to end with .rmeta" % build_metadata[0], + build_metadata[0].basename.endswith("_meta.rlib"), + "Expected %s to end with _meta.rlib" % build_metadata[0], ) asserts.equals(env, 1, len(rustc_rmeta_output), "Expected 1 rustc_rmeta_output file") diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 36a3de891b..a10dad44e2 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -2,7 +2,7 @@ load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") +load("//test/unit:common.bzl", "assert_argv_contains") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,49 +22,56 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Both actions should use the same --emit= - assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") - assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") + # The full action emits link; the metadata action emits only + # link with an explicit path (--emit=link=) and uses -Zno-codegen to + # produce a hollow rlib (metadata-full). + assert_argv_contains(env, rlib_action, "--emit=link") - # The metadata action should have a .rmeta as output and the rlib action a .rlib + # RustcMetadata uses --emit=link= to redirect the hollow rlib to the + # declared .rmeta output. Check that it contains the flag with a path. + metadata_emit = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=")] + asserts.true( + env, + len(metadata_emit) == 1, + "expected RustcMetadata to have --emit=link=, got " + str(metadata_emit), + ) + + # RustcMetadata must use -Zno-codegen to produce a hollow rlib + assert_argv_contains(env, metadata_action, "-Zno-codegen") + + # The metadata action outputs a hollow .rlib (_meta.rlib), the full action a normal .rlib path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib"), - "expected Rustc to output .rlib, got " + path, + path.endswith(".rlib") and not path.endswith("_meta.rlib"), + "expected Rustc to output .rlib (not _meta.rlib), got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rmeta"), - "expected RustcMetadata to output .rmeta, got " + path, + path.endswith("_meta.rlib"), + "expected RustcMetadata to output _meta.rlib, got " + path, ) - # Only the action building metadata should contain --rustc-quit-on-rmeta - assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # Check that both actions refer to the metadata of :first, not the rlib - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + # Both actions should refer to the metadata artifact of :first. + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("_meta.rlib")] asserts.true( env, len(extern_metadata) == 1, - "did not find a --extern=first=*.rmeta but expected one", + "expected RustcMetadata --extern=first=*_meta.rlib, got " + str([a for a in metadata_action.argv if "--extern=first=" in a]), ) - extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("_meta.rlib")] asserts.true( env, len(extern_rlib) == 1, - "did not find a --extern=first=*.rlib but expected one", + "expected Rustc --extern=first=*_meta.rlib, got " + str([a for a in rlib_action.argv if "--extern=first=" in a]), ) - # Check that the input to both actions is the metadata of :first - input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) - input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) - asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) + # Both actions should take the metadata artifact as input. + input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("_meta.rlib")] + asserts.true(env, len(input_metadata) == 1, "expected one libfirst _meta.rlib input to RustcMetadata, found " + str([i.path for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")])) + input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("_meta.rlib")] + asserts.true(env, len(input_rlib) == 1, "expected one libfirst _meta.rlib input to Rustc, found " + str([i.path for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")])) return analysistest.end(env) @@ -73,10 +80,10 @@ def _bin_test_impl(ctx): tut = analysistest.target_under_test(env) bin_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # Check that no inputs to this binary are .rmeta files. - metadata_inputs = [i.path for i in bin_action.inputs.to_list() if i.path.endswith(".rmeta")] + # Check that no inputs to this binary are hollow rlib (_meta.rlib) files. + metadata_inputs = [i.path for i in bin_action.inputs.to_list() if i.path.endswith("_meta.rlib")] - # Filter out toolchain targets. This test intends to only check for rmeta files of `deps`. + # Filter out toolchain targets. This test intends to only check for metadata files of `deps`. metadata_inputs = [i for i in metadata_inputs if "/lib/rustlib" not in i] asserts.false(env, metadata_inputs, "expected no metadata inputs, found " + json.encode_indent(metadata_inputs, indent = " " * 4)) @@ -130,6 +137,14 @@ def _pipelined_compilation_test(): ":bin_test", ] +def _is_metadata_file(path): + """Returns True if the path is a hollow rlib (metadata-full) file.""" + return path.endswith("_meta.rlib") + +def _is_full_rlib(path): + """Returns True if the path is a full rlib (not a hollow rlib).""" + return path.endswith(".rlib") and not path.endswith("_meta.rlib") + def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): env = analysistest.begin(ctx) tut = analysistest.target_under_test(env) @@ -138,24 +153,21 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] - seen_wrapper_metadata = False seen_to_wrap_metadata = False - for mi in metadata_inputs: - if "libwrapper" in mi.path: - seen_wrapper_metadata = True - if "libto_wrap" in mi.path: - seen_to_wrap_metadata = True - seen_wrapper_rlib = False seen_to_wrap_rlib = False - for ri in rlib_inputs: - if "libwrapper" in ri.path: - seen_wrapper_rlib = True - if "libto_wrap" in ri.path: - seen_to_wrap_rlib = True + for i in rust_action.inputs.to_list(): + if "libwrapper" in i.path: + if _is_metadata_file(i.path): + seen_wrapper_metadata = True + elif _is_full_rlib(i.path): + seen_wrapper_rlib = True + if "libto_wrap" in i.path: + if _is_metadata_file(i.path): + seen_to_wrap_metadata = True + elif _is_full_rlib(i.path): + seen_to_wrap_rlib = True if ctx.attr.generate_metadata: asserts.true(env, seen_wrapper_metadata, "expected dependency on metadata for 'wrapper' but not found") @@ -176,22 +188,23 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata + # The custom rule invocation should depend on metadata, regardless of whether + # the wrapper itself generates metadata. seen_to_wrap_rlib = False - seen_to_wrap_rmeta = False + seen_to_wrap_metadata = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith(".rlib"): + if "libto_wrap" in act.path and _is_full_rlib(act.path): seen_to_wrap_rlib = True - elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): - seen_to_wrap_rmeta = True + elif "libto_wrap" in act.path and _is_metadata_file(act.path): + seen_to_wrap_metadata = True - asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") + asserts.true(env, seen_to_wrap_metadata, "expected dependency on metadata for 'to_wrap' but not found") asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -249,6 +262,7 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, + generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index f24a0e421a..89b75b1850 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -44,7 +44,7 @@ def _wrap_impl(ctx): prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) tgt = ctx.attr.target diff --git a/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl b/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl index 1b0c353f61..f263a4538c 100644 --- a/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl +++ b/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl @@ -1,7 +1,7 @@ """Tests that rust_test targets receive codegen disambiguation flags. rust_test targets pass --codegen=metadata and --codegen=extra-filename to rustc -so that intermediate compilation artifacts (.o, .d files) get unique names. This +so that intermediate compilation artifacts (such as .o files) get unique names. This prevents collisions with rust_binary or rust_library targets that share the same crate name, which would otherwise cause link failures on non-sandboxed builds (e.g. Windows or --spawn_strategy=standalone). See diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 5d057b08cf..e484b0f4ed 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -541,47 +541,4 @@ mod test { Ok(()) } - #[test] - fn test_process_line_emit_link() -> Result<(), String> { - let mut metadata_emitted = false; - assert!(matches!( - process_line( - r#" - { - "$message_type": "artifact", - "emit": "link" - } - "# - .to_string(), - /*quit_on_rmeta=*/ true, - ErrorFormat::Rendered, - &mut metadata_emitted, - )?, - LineOutput::Skip - )); - assert!(!metadata_emitted); - Ok(()) - } - - #[test] - fn test_process_line_emit_metadata() -> Result<(), String> { - let mut metadata_emitted = false; - assert!(matches!( - process_line( - r#" - { - "$message_type": "artifact", - "emit": "metadata" - } - "# - .to_string(), - /*quit_on_rmeta=*/ true, - ErrorFormat::Rendered, - &mut metadata_emitted, - )?, - LineOutput::Terminate - )); - assert!(metadata_emitted); - Ok(()) - } } From 5529bac6eaf6700041a2b002baddff88582ecedd Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 2 Mar 2026 11:32:58 -0500 Subject: [PATCH 18/50] Cleanup some process_wrapper code --- rust/private/rustc.bzl | 3 +- util/process_wrapper/main.rs | 68 ++++----------------------------- util/process_wrapper/options.rs | 16 +------- util/process_wrapper/output.rs | 7 +--- util/process_wrapper/rustc.rs | 60 +++-------------------------- 5 files changed, 16 insertions(+), 138 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index bcfce3e981..7f9d145a01 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1140,8 +1140,7 @@ def construct_arguments( process_wrapper_flags.add("--rustc-output-format", "json" if error_format == "json" else "rendered") # Configure rustc json output by adding artifact notifications. - # These will always be filtered out by process_wrapper and will be use to terminate - # rustc when appropriate. + # These are filtered out by process_wrapper. json = ["artifacts"] if error_format == "short": json.append("diagnostic-short") diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index e484b0f4ed..8b10955464 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -25,7 +25,7 @@ use std::fmt; use std::fs::{self, copy, OpenOptions}; use std::io; use std::path::PathBuf; -use std::process::{exit, Command, ExitStatus, Stdio}; +use std::process::{exit, Command, Stdio}; #[cfg(windows)] use std::time::{SystemTime, UNIX_EPOCH}; @@ -37,30 +37,6 @@ use crate::rustc::ErrorFormat; #[cfg(windows)] use crate::util::read_file_to_array; -#[cfg(windows)] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On windows, there's no good way to know if the process was killed by a signal. - // If we killed the process, we override the code to signal success. - if was_killed { - 0 - } else { - status.code().unwrap_or(1) - } -} - -#[cfg(not(windows))] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On unix, if code is None it means that the process was killed by a signal. - // https://doc.rust-lang.org/std/process/struct.ExitStatus.html#method.success - match status.code() { - Some(code) => code, - // If we killed the process, we expect None here - None if was_killed => 0, - // Otherwise it's some unexpected signal - None => 1, - } -} - #[derive(Debug)] struct ProcessWrapperError(String); @@ -298,9 +274,7 @@ fn json_warning(line: &str) -> JsonValue { fn process_line( mut line: String, - quit_on_rmeta: bool, format: ErrorFormat, - metadata_emitted: &mut bool, ) -> Result { // LLVM can emit lines that look like the following, and these will be interspersed // with the regular JSON output. Arguably, rustc should be fixed not to emit lines @@ -315,11 +289,7 @@ fn process_line( return Ok(LineOutput::Skip); } } - if quit_on_rmeta { - rustc::stop_on_rmeta_completion(line, format, metadata_emitted) - } else { - rustc::process_json(line, format) - } + rustc::process_json(line, format) } fn main() -> Result<(), ProcessWrapperError> { @@ -381,26 +351,13 @@ fn main() -> Result<(), ProcessWrapperError> { None }; - let mut was_killed = false; let result = if let Some(format) = opts.rustc_output_format { - let quit_on_rmeta = opts.rustc_quit_on_rmeta; - // Process json rustc output and kill the subprocess when we get a signal - // that we emitted a metadata file. - let mut me = false; - let metadata_emitted = &mut me; - let result = process_output( + process_output( &mut child_stderr, stderr.as_mut(), output_file.as_mut(), - move |line| process_line(line, quit_on_rmeta, format, metadata_emitted), - ); - if me { - // If recv returns Ok(), a signal was sent in this channel so we should terminate the child process. - // We can safely ignore the Result from kill() as we don't care if the process already terminated. - let _ = child.kill(); - was_killed = true; - } - result + move |line| process_line(line, format), + ) } else { // Process output normally by forwarding stderr process_output( @@ -415,10 +372,8 @@ fn main() -> Result<(), ProcessWrapperError> { let status = child .wait() .map_err(|e| ProcessWrapperError(format!("failed to wait for child process: {}", e)))?; - // If the child process is rustc and is killed after metadata generation, that's also a success. - let code = status_code(status, was_killed); - let success = code == 0; - if success { + let code = status.code().unwrap_or(1); + if code == 0 { if let Some(tf) = opts.touch_file { OpenOptions::new() .create(true) @@ -454,7 +409,6 @@ mod test { #[test] fn test_process_line_diagnostic_json() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -463,9 +417,7 @@ mod test { } "# .to_string(), - false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -486,7 +438,6 @@ mod test { #[test] fn test_process_line_diagnostic_rendered() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -495,9 +446,7 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Rendered, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -508,16 +457,13 @@ mod test { #[test] fn test_process_line_noise() -> Result<(), String> { - let mut metadata_emitted = false; for text in [ "'+zaamo' is not a recognized feature for this target (ignoring feature)", " WARN rustc_errors::emitter Invalid span...", ] { let LineOutput::Message(msg) = process_line( text.to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); diff --git a/util/process_wrapper/options.rs b/util/process_wrapper/options.rs index 4416fb220b..7ec7b535ab 100644 --- a/util/process_wrapper/options.rs +++ b/util/process_wrapper/options.rs @@ -44,9 +44,6 @@ pub(crate) struct Options { // If set, also logs all unprocessed output from the rustc output to this file. // Meant to be used to get json output out of rustc for tooling usage. pub(crate) output_file: Option, - // If set, it configures rustc to emit an rmeta file and then - // quit. - pub(crate) rustc_quit_on_rmeta: bool, // This controls the output format of rustc messages. pub(crate) rustc_output_format: Option, } @@ -65,7 +62,6 @@ pub(crate) fn options() -> Result { let mut stdout_file = None; let mut stderr_file = None; let mut output_file = None; - let mut rustc_quit_on_rmeta_raw = None; let mut rustc_output_format_raw = None; let mut flags = Flags::new(); let mut require_explicit_unstable_features = None; @@ -113,17 +109,9 @@ pub(crate) fn options() -> Result { "Log all unprocessed subprocess stderr in this file.", &mut output_file, ); - flags.define_flag( - "--rustc-quit-on-rmeta", - "If enabled, this wrapper will terminate rustc after rmeta has been emitted.", - &mut rustc_quit_on_rmeta_raw, - ); flags.define_flag( "--rustc-output-format", - "Controls the rustc output format if --rustc-quit-on-rmeta is set.\n\ - 'json' will cause the json output to be output, \ - 'rendered' will extract the rendered message and print that.\n\ - Default: `rendered`", + "The expected rustc output format. Valid values: json, rendered.", &mut rustc_output_format_raw, ); flags.define_flag( @@ -251,7 +239,6 @@ pub(crate) fn options() -> Result { }) .transpose()?; - let rustc_quit_on_rmeta = rustc_quit_on_rmeta_raw.is_some_and(|s| s == "true"); let rustc_output_format = rustc_output_format_raw .map(|v| match v.as_str() { "json" => Ok(rustc::ErrorFormat::Json), @@ -299,7 +286,6 @@ pub(crate) fn options() -> Result { stdout_file, stderr_file, output_file, - rustc_quit_on_rmeta, rustc_output_format, }) } diff --git a/util/process_wrapper/output.rs b/util/process_wrapper/output.rs index 4b3604b18d..5dabad8179 100644 --- a/util/process_wrapper/output.rs +++ b/util/process_wrapper/output.rs @@ -18,15 +18,11 @@ use std::io::{self, prelude::*}; /// LineOutput tells process_output what to do when a line is processed. /// If a Message is returned, it will be written to write_end, if -/// Skip is returned nothing will be printed and execution continues, -/// if Terminate is returned, process_output returns immediately. -/// Terminate is used to stop processing when we see an emit metadata -/// message. +/// Skip is returned nothing will be printed and execution continues. #[derive(Debug)] pub(crate) enum LineOutput { Message(String), Skip, - Terminate, } #[derive(Debug)] @@ -95,7 +91,6 @@ where match process_line(line.clone()) { Ok(LineOutput::Message(to_write)) => output_writer.write_all(to_write.as_bytes())?, Ok(LineOutput::Skip) => {} - Ok(LineOutput::Terminate) => return Ok(()), Err(msg) => { failed_on = Some((line, msg)); break; diff --git a/util/process_wrapper/rustc.rs b/util/process_wrapper/rustc.rs index 97ee466337..3bb4a8c2d9 100644 --- a/util/process_wrapper/rustc.rs +++ b/util/process_wrapper/rustc.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::convert::{TryFrom, TryInto}; - use tinyjson::JsonValue; use crate::output::{LineOutput, LineResult}; @@ -37,66 +35,20 @@ fn get_key(value: &JsonValue, key: &str) -> Option { } } -#[derive(Debug)] -enum RustcMessage { - Emit(String), - Message(String), -} - -impl TryFrom for RustcMessage { - type Error = (); - fn try_from(val: JsonValue) -> Result { - if let Some(emit) = get_key(&val, "emit") { - return Ok(Self::Emit(emit)); - } - if let Some(rendered) = get_key(&val, "rendered") { - return Ok(Self::Message(rendered)); - } - Err(()) - } -} - /// process_rustc_json takes an output line from rustc configured with /// --error-format=json, parses the json and returns the appropriate output /// according to the original --error-format supplied. -/// Only messages are returned, emits are ignored. +/// Only diagnostics with a rendered message are returned. /// Returns an errors if parsing json fails. pub(crate) fn process_json(line: String, error_format: ErrorFormat) -> LineResult { let parsed: JsonValue = line .parse() .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, - }) -} - -/// stop_on_rmeta_completion parses the json output of rustc in the same way -/// process_rustc_json does. In addition, it will signal to stop when metadata -/// is emitted so the compiler can be terminated. -/// This is used to implement pipelining in rules_rust, please see -/// https://internals.rust-lang.org/t/evaluating-pipelined-rustc-compilation/10199 -/// Returns an error if parsing json fails. -/// TODO: pass a function to handle the emit event and merge with process_json -pub(crate) fn stop_on_rmeta_completion( - line: String, - error_format: ErrorFormat, - kill: &mut bool, -) -> LineResult { - let parsed: JsonValue = line - .parse() - .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Emit(emit)) if emit == "metadata" => { - *kill = true; - LineOutput::Terminate - } - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, + Ok(if let Some(rendered) = get_key(&parsed, "rendered") { + output_based_on_error_format(line, rendered, error_format) + } else { + // Ignore non-diagnostic messages such as artifact notifications. + LineOutput::Skip }) } From 10b6860769f073a0c6fd9077ff292be213498b37 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 13 Mar 2026 14:19:28 -0400 Subject: [PATCH 19/50] Fix prost to be compatible with multiplatform --- extensions/prost/private/BUILD.bazel | 2 ++ extensions/prost/private/prost.bzl | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/extensions/prost/private/BUILD.bazel b/extensions/prost/private/BUILD.bazel index ed8e508537..aaf6c22d8d 100644 --- a/extensions/prost/private/BUILD.bazel +++ b/extensions/prost/private/BUILD.bazel @@ -5,6 +5,8 @@ load("//:defs.bzl", "rust_prost_toolchain") load(":legacy_proto_toolchain.bzl", "legacy_proto_toolchain") load(":prost.bzl", "RUST_EDITION", "current_prost_runtime") +exports_files(["protoc_wrapper.rs"]) + current_prost_runtime( name = "current_prost_runtime", ) diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index 6a8d36e10d..e67ff89fcf 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -31,14 +31,14 @@ RUST_EDITION = "2021" TOOLCHAIN_TYPE = "@rules_rust_prost//:toolchain_type" -def _create_proto_lang_toolchain(ctx, prost_toolchain): +def _create_proto_lang_toolchain(prost_toolchain): proto_lang_toolchain = proto_common.ProtoLangToolchainInfo( out_replacement_format_flag = "--prost_out=%s", plugin_format_flag = prost_toolchain.prost_plugin_flag, plugin = prost_toolchain.prost_plugin[DefaultInfo].files_to_run, runtime = prost_toolchain.prost_runtime, provided_proto_sources = depset(), - proto_compiler = ctx.attr._prost_process_wrapper[DefaultInfo].files_to_run, + proto_compiler = prost_toolchain.prost_process_wrapper[DefaultInfo].files_to_run, protoc_opts = prost_toolchain.protoc_opts, progress_message = "ProstGenProto %{label}", mnemonic = "ProstGenProto", @@ -118,7 +118,7 @@ def _compile_proto( additional_inputs = additional_inputs, additional_args = additional_args, generated_files = [lib_rs, package_info_file], - proto_lang_toolchain_info = _create_proto_lang_toolchain(ctx, prost_toolchain), + proto_lang_toolchain_info = _create_proto_lang_toolchain(prost_toolchain), plugin_output = ctx.bin_dir.path, ) @@ -377,12 +377,6 @@ rust_prost_aspect = aspect( default = Label("@bazel_tools//tools/cpp:grep-includes"), cfg = "exec", ), - "_prost_process_wrapper": attr.label( - doc = "The wrapper script for the Prost protoc plugin.", - cfg = "exec", - executable = True, - default = Label("//private:protoc_wrapper"), - ), } | RUSTC_ATTRS | { # Need to override this attribute to explicitly set the workspace. "_always_enable_metadata_output_groups": attr.label( @@ -473,6 +467,7 @@ def _rust_prost_toolchain_impl(ctx): prost_plugin = ctx.attr.prost_plugin, prost_plugin_flag = ctx.attr.prost_plugin_flag, prost_runtime = ctx.attr.prost_runtime, + prost_process_wrapper = ctx.attr._prost_process_wrapper, prost_types = ctx.attr.prost_types, proto_compiler = proto_compiler, protoc_opts = ctx.fragments.proto.experimental_protoc_opts, @@ -516,6 +511,12 @@ rust_prost_toolchain = rule( mandatory = True, aspects = [rust_analyzer_aspect], ), + "_prost_process_wrapper": attr.label( + doc = "The wrapper script for the Prost protoc plugin.", + cfg = "exec", + executable = True, + default = Label("@rules_rust_prost//private:protoc_wrapper"), + ), "prost_types": attr.label( doc = "The Prost types crates to use.", providers = [[rust_common.crate_info], [rust_common.crate_group_info]], From b430aa6e731a4891e5ac92417af397e315c23fe9 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 5 Apr 2026 15:59:46 -0400 Subject: [PATCH 20/50] Improve toolchain make var env expansion --- rust/private/rust.bzl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 496cabb000..9ec672a4cc 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -59,6 +59,14 @@ load( # TODO(marco): Separate each rule into its own file. +def _toolchain_make_variables(ctx): + make_variables = {} + for toolchain_target in getattr(ctx.attr, "toolchains", []): + if platform_common.TemplateVariableInfo in toolchain_target: + variables = getattr(toolchain_target[platform_common.TemplateVariableInfo], "variables", {}) + make_variables.update(variables) + return make_variables + def _assert_no_deprecated_attributes(_ctx): """Forces a failure if any deprecated attributes were specified @@ -285,7 +293,7 @@ def _rust_binary_impl(ctx): ctx, ctx.attr.env, ctx.attr.data, - {}, + _toolchain_make_variables(ctx), ), )) @@ -472,7 +480,7 @@ def _rust_test_impl(ctx): ctx, getattr(ctx.attr, "env", {}), data, - {}, + _toolchain_make_variables(ctx), ) if toolchain.coverage_supported and ctx.configuration.coverage_enabled: if not toolchain.llvm_profdata: From a5ffc2b23b3415fea435207ad0b7cc1c8c571413 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 9 Apr 2026 11:22:27 -0400 Subject: [PATCH 21/50] Improve rust compilation messages --- cargo/private/cargo_build_script.bzl | 2 +- rust/private/rustc.bzl | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_script.bzl index 301e2a8dce..9e68298330 100644 --- a/cargo/private/cargo_build_script.bzl +++ b/cargo/private/cargo_build_script.bzl @@ -642,7 +642,7 @@ def _cargo_build_script_impl(ctx): tools = tools, inputs = depset(build_script_inputs, transitive = [runfiles_inputs]), mnemonic = "CargoBuildScriptRun", - progress_message = "Running Cargo build script {}".format(pkg_name), + progress_message = "Running Cargo build script %{label}", env = env, toolchain = None, use_default_shell_env = use_default_shell_env, diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 7f9d145a01..0fc3a78a7a 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1716,9 +1716,8 @@ def rustc_compile_action( env = env, arguments = args.all, mnemonic = "Rustc", - progress_message = "Compiling Rust {} {}{} ({} file{})".format( + progress_message = "Compiling Rust {} %{{label}}{} ({} file{})".format( crate_info.type, - ctx.label.name, formatted_version, len(srcs), "" if len(srcs) == 1 else "s", @@ -1735,9 +1734,8 @@ def rustc_compile_action( env = env, arguments = args_metadata.all, mnemonic = "RustcMetadata", - progress_message = "Compiling Rust metadata {} {}{} ({} file{})".format( + progress_message = "Compiling Rust metadata {} %{{label}}{} ({} file{})".format( crate_info.type, - ctx.label.name, formatted_version, len(srcs), "" if len(srcs) == 1 else "s", @@ -1756,9 +1754,8 @@ def rustc_compile_action( env = env, arguments = [args.rustc_path, args.rustc_flags], mnemonic = "Rustc", - progress_message = "Compiling Rust (without process_wrapper) {} {}{} ({} file{})".format( + progress_message = "Compiling Rust (without process_wrapper) {} %{{label}}{} ({} file{})".format( crate_info.type, - ctx.label.name, formatted_version, len(srcs), "" if len(srcs) == 1 else "s", From 59278d6cb6b09c2dfd284190c88dbbbea72d8898 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 15 Apr 2026 13:21:12 -0700 Subject: [PATCH 22/50] Add rust_test sharding support (#13) Port the sharding wrapper feature from bazelbuild/rules_rust#3774 into the hermeticbuild fork. The implementation wraps rust_test executables when experimental_enable_sharding is set while keeping rustc_compile_action's existing provider-list API for internal and extension callers. rust_test now scans the returned providers to replace DefaultInfo for the wrapper, so extensions such as prost and wasm-bindgen continue to consume rustc_compile_action without API churn. Co-authored-by: Brian Duff Co-authored-by: Codex --- rust/private/BUILD.bazel | 5 + rust/private/rust.bzl | 91 +++++++++ rust/private/test_sharding_wrapper.bat | 192 ++++++++++++++++++ rust/private/test_sharding_wrapper.sh | 90 ++++++++ test/unit/test_sharding/BUILD.bazel | 4 + .../unit/test_sharding/fake_libtest_binary.sh | 67 ++++++ test/unit/test_sharding/sharded_test.rs | 39 ++++ test/unit/test_sharding/test_sharding.bzl | 100 +++++++++ ...st_sharding_wrapper_hashes_sorted_names.sh | 73 +++++++ 9 files changed, 661 insertions(+) create mode 100644 rust/private/test_sharding_wrapper.bat create mode 100644 rust/private/test_sharding_wrapper.sh create mode 100644 test/unit/test_sharding/BUILD.bazel create mode 100755 test/unit/test_sharding/fake_libtest_binary.sh create mode 100644 test/unit/test_sharding/sharded_test.rs create mode 100644 test/unit/test_sharding/test_sharding.bzl create mode 100755 test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index d3cea4650d..4a9afc2995 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -6,6 +6,11 @@ load("//rust/private:stamp.bzl", "stamp_build_setting") # Exported for docs exports_files(["providers.bzl"]) +exports_files([ + "test_sharding_wrapper.bat", + "test_sharding_wrapper.sh", +]) + bzl_library( name = "bazel_tools_bzl_lib", srcs = ["@bazel_tools//tools:bzl_srcs"], diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 9ec672a4cc..75733e7d0d 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -474,6 +474,7 @@ def _rust_test_impl(ctx): rust_flags = get_rust_test_flags(ctx.attr), skip_expanding_rustc_env = True, ) + providers = _maybe_wrap_sharded_test(ctx, providers, toolchain) data = getattr(ctx.attr, "data", []) env = expand_dict_value_locations( @@ -516,6 +517,70 @@ def _rust_test_impl(ctx): return providers +def _maybe_wrap_sharded_test(ctx, providers, toolchain): + if not ctx.attr.experimental_enable_sharding or not ctx.attr.use_libtest_harness: + return providers + + crate_info = _find_test_crate_info(providers) + if crate_info == None: + fail("Couldn't find crate_info or test_crate_info in rust_test providers") + test_binary = crate_info.output + wrapper, wrapper_template = _declare_test_sharding_wrapper(ctx, toolchain) + + ctx.actions.expand_template( + template = wrapper_template, + output = wrapper, + substitutions = { + "{{TEST_BINARY}}": test_binary.short_path, + }, + is_executable = True, + ) + + wrapped_providers = [] + replaced_default_info = False + for provider in providers: + if _is_default_info(provider): + wrapped_providers.append(DefaultInfo( + files = provider.files, + runfiles = provider.default_runfiles.merge(ctx.runfiles(files = [test_binary])), + executable = wrapper, + )) + replaced_default_info = True + else: + wrapped_providers.append(provider) + + if not replaced_default_info: + fail("Couldn't find DefaultInfo in rust_test providers") + + return wrapped_providers + +def _find_test_crate_info(providers): + for provider in providers: + if hasattr(provider, "crate"): + return provider.crate + if hasattr(provider, "name"): + return provider + return None + +def _is_default_info(provider): + return ( + hasattr(provider, "default_runfiles") and + hasattr(provider, "files") and + hasattr(provider, "files_to_run") + ) + +def _declare_test_sharding_wrapper(ctx, toolchain): + if toolchain.target_os == "windows": + return ( + ctx.actions.declare_file(ctx.label.name + "_sharding_wrapper.bat"), + ctx.file._test_sharding_wrapper_windows, + ) + + return ( + ctx.actions.declare_file(ctx.label.name + "_sharding_wrapper.sh"), + ctx.file._test_sharding_wrapper_unix, + ) + def _rust_library_group_impl(ctx): dep_variant_infos = [] dep_variant_transitive_infos = [] @@ -892,6 +957,24 @@ _RUST_TEST_ATTRS = { "env_inherit": attr.string_list( doc = "Specifies additional environment variables to inherit from the external environment when the test is executed by bazel test.", ), + "experimental_enable_sharding": attr.bool( + mandatory = False, + default = False, + doc = dedent("""\ + If True, enable support for Bazel test sharding (shard_count attribute). + + When enabled, tests are executed via a wrapper script that: + 1. Enumerates tests using libtest's --list flag + 2. Sorts tests by name and partitions them across shards by stable name hash + 3. Uses either Bazel's native TEST_TOTAL_SHARDS/TEST_SHARD_INDEX env + or explicit RULES_RUST_TEST_TOTAL_SHARDS/RULES_RUST_TEST_SHARD_INDEX env + 4. Runs only the tests assigned to the current shard + + This attribute only has an effect when use_libtest_harness is True. + + This is experimental and may change in future releases. + """), + ), "use_libtest_harness": attr.bool( mandatory = False, default = True, @@ -901,6 +984,14 @@ _RUST_TEST_ATTRS = { E.g. `bazel test //src:rust_test --test_arg=foo::test::test_fn`. """), ), + "_test_sharding_wrapper_unix": attr.label( + default = Label("//rust/private:test_sharding_wrapper.sh"), + allow_single_file = True, + ), + "_test_sharding_wrapper_windows": attr.label( + default = Label("//rust/private:test_sharding_wrapper.bat"), + allow_single_file = True, + ), } | _COVERAGE_ATTRS | _EXPERIMENTAL_USE_CC_COMMON_LINK_ATTRS rust_library = rule( diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat new file mode 100644 index 0000000000..e90a803f1c --- /dev/null +++ b/rust/private/test_sharding_wrapper.bat @@ -0,0 +1,192 @@ +@REM Copyright 2024 The Bazel Authors. All rights reserved. +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@REM Wrapper script for rust_test that enables Bazel test sharding support. +@REM This script intercepts test execution, enumerates tests using libtest's +@REM --list flag, partitions them by stable test-name hash, and runs only the +@REM relevant subset. + +@ECHO OFF +SETLOCAL EnableDelayedExpansion + +SET TEST_BINARY_RAW={{TEST_BINARY}} +SET TEST_BINARY_PATH=!TEST_BINARY_RAW:/=\! + +@REM Try to find the binary using RUNFILES_DIR if set +IF DEFINED RUNFILES_DIR ( + SET TEST_BINARY_IN_RUNFILES=!RUNFILES_DIR!\!TEST_BINARY_PATH! + IF EXIST "!TEST_BINARY_IN_RUNFILES!" ( + SET TEST_BINARY_PATH=!TEST_BINARY_IN_RUNFILES! + ) +) + +@REM The short_path is like: test/unit/test_sharding/test-2586318641/sharded_test_enabled.exe +@REM But on Windows, the binary is at grandparent/test-XXX/name.exe (sibling of runfiles dir) +@REM Extract just the last two components (test-XXX/name.exe) +FOR %%F IN ("!TEST_BINARY_PATH!") DO SET BINARY_NAME=%%~nxF +FOR %%F IN ("!TEST_BINARY_PATH!\..") DO SET BINARY_DIR=%%~nxF + +@REM Try various path resolutions +SET FOUND_BINARY=0 + +@REM Try 1: Direct path (might work in some configurations) +IF EXIST "!TEST_BINARY_PATH!" ( + SET FOUND_BINARY=1 +) + +@REM Try 2: Grandparent + last two path components +IF !FOUND_BINARY! EQU 0 ( + FOR %%F IN ("!TEST_BINARY_PATH!") DO ( + SET TEMP_PATH=%%~dpF + SET TEMP_PATH=!TEMP_PATH:~0,-1! + FOR %%D IN ("!TEMP_PATH!") DO SET PARENT_DIR=%%~nxD + ) + SET TEST_BINARY_GP=..\..\!PARENT_DIR!\!BINARY_NAME! + IF EXIST "!TEST_BINARY_GP!" ( + SET TEST_BINARY_PATH=!TEST_BINARY_GP! + SET FOUND_BINARY=1 + ) +) + +@REM Try 3: RUNFILES_DIR based path +IF !FOUND_BINARY! EQU 0 IF DEFINED RUNFILES_DIR ( + SET TEST_BINARY_RF=!RUNFILES_DIR!\_main\!TEST_BINARY_PATH! + SET TEST_BINARY_RF=!TEST_BINARY_RF:/=\! + IF EXIST "!TEST_BINARY_RF!" ( + SET TEST_BINARY_PATH=!TEST_BINARY_RF! + SET FOUND_BINARY=1 + ) +) + +@REM Try 4: manifest-based runfile lookup. This covers nested launchers that +@REM execute the sharding wrapper from another test's runfiles tree. +IF !FOUND_BINARY! EQU 0 ( + SET "MANIFEST=!RUNFILES_MANIFEST_FILE!" + IF NOT DEFINED MANIFEST IF EXIST "%~f0.runfiles_manifest" SET "MANIFEST=%~f0.runfiles_manifest" + IF NOT DEFINED MANIFEST IF EXIST "%~dpn0.runfiles_manifest" SET "MANIFEST=%~dpn0.runfiles_manifest" + IF NOT DEFINED MANIFEST IF EXIST "%~f0.exe.runfiles_manifest" SET "MANIFEST=%~f0.exe.runfiles_manifest" + + IF DEFINED MANIFEST IF EXIST "!MANIFEST!" ( + SET "TEST_BINARY_MANIFEST_PATH=!TEST_BINARY_RAW!" + SET "TEST_BINARY_MANIFEST_PATH=!TEST_BINARY_MANIFEST_PATH:\=/!" + IF DEFINED TEST_WORKSPACE SET "TEST_BINARY_MANIFEST_WORKSPACE_PATH=!TEST_WORKSPACE!/!TEST_BINARY_MANIFEST_PATH!" + FOR /F "usebackq tokens=1,* delims= " %%A IN ("!MANIFEST!") DO ( + IF "%%A"=="!TEST_BINARY_MANIFEST_PATH!" ( + SET "TEST_BINARY_PATH=%%B" + SET FOUND_BINARY=1 + GOTO :FOUND_TEST_BINARY + ) + IF DEFINED TEST_BINARY_MANIFEST_WORKSPACE_PATH IF "%%A"=="!TEST_BINARY_MANIFEST_WORKSPACE_PATH!" ( + SET "TEST_BINARY_PATH=%%B" + SET FOUND_BINARY=1 + GOTO :FOUND_TEST_BINARY + ) + ) + ) +) + +:FOUND_TEST_BINARY + +IF !FOUND_BINARY! EQU 0 ( + ECHO ERROR: Could not find test binary at any expected location + EXIT /B 1 +) + +@REM Native Bazel test sharding sets TEST_TOTAL_SHARDS/TEST_SHARD_INDEX. +@REM Explicit shard test targets can set RULES_RUST_TEST_TOTAL_SHARDS/ +@REM RULES_RUST_TEST_SHARD_INDEX instead because Bazel may reserve TEST_* +@REM variables for its own test runner env. +SET TOTAL_SHARDS=%RULES_RUST_TEST_TOTAL_SHARDS% +IF "%TOTAL_SHARDS%"=="" SET TOTAL_SHARDS=%TEST_TOTAL_SHARDS% +SET SHARD_INDEX=%RULES_RUST_TEST_SHARD_INDEX% +IF "%SHARD_INDEX%"=="" SET SHARD_INDEX=%TEST_SHARD_INDEX% + +@REM If sharding is not enabled, run test binary directly +IF "%TOTAL_SHARDS%"=="" ( + !TEST_BINARY_PATH! %* + EXIT /B !ERRORLEVEL! +) +IF "%TOTAL_SHARDS%"=="0" ( + !TEST_BINARY_PATH! %* + EXIT /B !ERRORLEVEL! +) + +IF "%SHARD_INDEX%"=="" ( + ECHO ERROR: TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled + EXIT /B 1 +) + +@REM Touch status file to advertise sharding support to Bazel +IF NOT "%TEST_SHARD_STATUS_FILE%"=="" IF NOT "%TEST_TOTAL_SHARDS%"=="" IF NOT "%TEST_TOTAL_SHARDS%"=="0" ( + TYPE NUL > "%TEST_SHARD_STATUS_FILE%" +) + +@REM Create per-wrapper temporary files. Prefer Bazel's per-test temp directory; +@REM when falling back to the shared temp directory, avoid %RANDOM%-only file +@REM names that can collide across concurrently running Windows test shards. +SET "TEMP_ROOT=%TEST_TMPDIR%" +IF NOT DEFINED TEMP_ROOT SET "TEMP_ROOT=%TEMP%" +IF NOT DEFINED TEMP_ROOT SET "TEMP_ROOT=." +:CREATE_TEMP_DIR +SET "TEMP_DIR=!TEMP_ROOT!\rust_test_sharding_!RANDOM!_!RANDOM!_!RANDOM!" +MKDIR "!TEMP_DIR!" 2>NUL +IF ERRORLEVEL 1 GOTO :CREATE_TEMP_DIR +SET "TEMP_LIST=!TEMP_DIR!\list.txt" +SET "TEMP_SHARD_LIST=!TEMP_DIR!\shard.txt" + +@REM Enumerate all tests using libtest's --list flag +!TEST_BINARY_PATH! --list --format terse 2>NUL > "!TEMP_LIST!" +IF ERRORLEVEL 1 ( + RMDIR /S /Q "!TEMP_DIR!" 2>NUL + EXIT /B 1 +) + +@REM Sort tests by ordinal name and filter this shard by stable FNV-1a hash so +@REM adding or removing one test does not move unrelated tests between shards. +@REM In the PowerShell fragment below, 2166136261 is the 32-bit FNV offset basis, +@REM 16777619 is the FNV prime, and 4294967295 is the UInt32 mask. Use decimal +@REM constants because Windows PowerShell can interpret 0xffffffff as -1. +powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ + "$ErrorActionPreference = 'Stop';" ^ + "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ + "[Array]::Sort($tests, [StringComparer]::Ordinal);" ^ + "$totalShards = [uint32]$env:TOTAL_SHARDS; $shardIndex = [uint32]$env:SHARD_INDEX;" ^ + "$fnvPrime = [uint64]16777619; $u32Mask = [uint64]4294967295;" ^ + "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32](([uint64]($hash -bxor $byte) * $fnvPrime) -band $u32Mask) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ + > "!TEMP_SHARD_LIST!" +IF ERRORLEVEL 1 ( + RMDIR /S /Q "!TEMP_DIR!" 2>NUL + EXIT /B 1 +) + +SET SHARD_TESTS= + +FOR /F "usebackq delims=" %%T IN ("!TEMP_SHARD_LIST!") DO ( + IF "!SHARD_TESTS!"=="" ( + SET SHARD_TESTS=%%T + ) ELSE ( + SET SHARD_TESTS=!SHARD_TESTS! %%T + ) +) + +RMDIR /S /Q "!TEMP_DIR!" 2>NUL + +@REM If no tests for this shard, exit successfully +IF "!SHARD_TESTS!"=="" ( + EXIT /B 0 +) + +@REM Run the filtered tests with --exact to match exact test names +!TEST_BINARY_PATH! !SHARD_TESTS! --exact %* +EXIT /B !ERRORLEVEL! diff --git a/rust/private/test_sharding_wrapper.sh b/rust/private/test_sharding_wrapper.sh new file mode 100644 index 0000000000..b1f0fb55d7 --- /dev/null +++ b/rust/private/test_sharding_wrapper.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper script for rust_test that enables Bazel test sharding support. +# This script intercepts test execution, enumerates tests using libtest's +# --list flag, partitions them by stable test-name hash, and runs only the +# relevant subset. + +set -euo pipefail + +TEST_BINARY="{{TEST_BINARY}}" +# Native Bazel test sharding sets TEST_TOTAL_SHARDS/TEST_SHARD_INDEX. Explicit +# shard test targets can set RULES_RUST_TEST_TOTAL_SHARDS/RULES_RUST_TEST_SHARD_INDEX +# instead because Bazel may reserve TEST_* variables for its own test runner env. +TOTAL_SHARDS="${RULES_RUST_TEST_TOTAL_SHARDS:-${TEST_TOTAL_SHARDS:-}}" +SHARD_INDEX="${RULES_RUST_TEST_SHARD_INDEX:-${TEST_SHARD_INDEX:-}}" + +test_shard_index() { + local test_name="$1" + # FNV-1a 32-bit hash. The initial value is the FNV offset basis, and + # 16777619 is the FNV prime. This gives a stable, cheap string hash without + # depending on platform-specific tools being present in the test sandbox. + local hash=2166136261 + local byte + local char + local i + local LC_ALL=C + + for ((i = 0; i < ${#test_name}; i++)); do + char="${test_name:i:1}" + printf -v byte "%d" "'$char" + hash=$(( ((hash ^ byte) * 16777619) & 0xffffffff )) + done + + echo $(( hash % TOTAL_SHARDS )) +} + +# If sharding is not enabled, run test binary directly +if [[ -z "${TOTAL_SHARDS}" || "${TOTAL_SHARDS}" == "0" ]]; then + exec "./${TEST_BINARY}" "$@" +fi + +if [[ -z "${SHARD_INDEX}" ]]; then + echo "TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled" >&2 + exit 1 +fi + +# Touch status file to advertise sharding support to Bazel +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" && "${TEST_TOTAL_SHARDS:-0}" != "0" ]]; then + touch "${TEST_SHARD_STATUS_FILE}" +fi + +# Enumerate all tests using libtest's --list flag. Sort the list so execution +# order does not depend on libtest's output order. +# Output format: "test_name: test" - we need to strip the ": test" suffix +test_list=$("./${TEST_BINARY}" --list --format terse 2>/dev/null | grep ': test$' | sed 's/: test$//' | LC_ALL=C sort || true) + +# If no tests found, exit successfully +if [[ -z "$test_list" ]]; then + exit 0 +fi + +# Filter tests for this shard. Use a stable name hash instead of list position +# so adding or removing one test does not move unrelated tests between shards. +shard_tests=() +while IFS= read -r test_name; do + if (( $(test_shard_index "$test_name") == SHARD_INDEX )); then + shard_tests+=("$test_name") + fi +done <<< "$test_list" + +# If no tests for this shard, exit successfully +if [[ ${#shard_tests[@]} -eq 0 ]]; then + exit 0 +fi + +# Run the filtered tests with --exact to match exact test names +exec "./${TEST_BINARY}" "${shard_tests[@]}" --exact "$@" diff --git a/test/unit/test_sharding/BUILD.bazel b/test/unit/test_sharding/BUILD.bazel new file mode 100644 index 0000000000..0fbfffb43f --- /dev/null +++ b/test/unit/test_sharding/BUILD.bazel @@ -0,0 +1,4 @@ +load(":test_sharding.bzl", "test_sharding_test_suite") + +############################ UNIT TESTS ############################# +test_sharding_test_suite(name = "test_sharding_test_suite") diff --git a/test/unit/test_sharding/fake_libtest_binary.sh b/test/unit/test_sharding/fake_libtest_binary.sh new file mode 100755 index 0000000000..fef029be5c --- /dev/null +++ b/test/unit/test_sharding/fake_libtest_binary.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +emit_base_tests() { + cat <<'EOF' +delta::test_delta: test +alpha::test_alpha: test +foxtrot::test_foxtrot: test +bravo::test_bravo: test +echo::test_echo: test +charlie::test_charlie: test +helper::bench: bench +EOF +} + +emit_reversed_base_tests() { + cat <<'EOF' +helper::bench: bench +charlie::test_charlie: test +echo::test_echo: test +bravo::test_bravo: test +foxtrot::test_foxtrot: test +alpha::test_alpha: test +delta::test_delta: test +EOF +} + +emit_tests_with_added_test() { + cat <<'EOF' +delta::test_delta: test +alpha::test_alpha: test +foxtrot::test_foxtrot: test +aardvark::test_added: test +bravo::test_bravo: test +echo::test_echo: test +charlie::test_charlie: test +helper::bench: bench +EOF +} + +if [[ "${1:-}" == "--list" ]]; then + case "${TEST_LIST_VARIANT:-base}:${TEST_LIST_ORDER:-normal}" in + base:normal) + emit_base_tests + ;; + base:reversed) + emit_reversed_base_tests + ;; + with_added:normal) + emit_tests_with_added_test + ;; + *) + echo "unknown test list variant: ${TEST_LIST_VARIANT:-base}:${TEST_LIST_ORDER:-normal}" >&2 + exit 1 + ;; + esac + exit 0 +fi + +: "${TEST_SHARD_OUTPUT:?}" + +for test_name in "$@"; do + if [[ "$test_name" != "--exact" ]]; then + printf '%s\n' "$test_name" >> "$TEST_SHARD_OUTPUT" + fi +done diff --git a/test/unit/test_sharding/sharded_test.rs b/test/unit/test_sharding/sharded_test.rs new file mode 100644 index 0000000000..322336c202 --- /dev/null +++ b/test/unit/test_sharding/sharded_test.rs @@ -0,0 +1,39 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test file with multiple tests for verifying sharding support. +//! +//! These tests are intentionally trivial. Their purpose is to provide multiple +//! enumerable test functions that can be partitioned across shards. + +#[cfg(test)] +mod tests { + #[test] + fn test_1() {} + + #[test] + fn test_2() {} + + #[test] + fn test_3() {} + + #[test] + fn test_4() {} + + #[test] + fn test_5() {} + + #[test] + fn test_6() {} +} diff --git a/test/unit/test_sharding/test_sharding.bzl b/test/unit/test_sharding/test_sharding.bzl new file mode 100644 index 0000000000..14005f7160 --- /dev/null +++ b/test/unit/test_sharding/test_sharding.bzl @@ -0,0 +1,100 @@ +"""Tests for rust_test sharding support.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//rust:defs.bzl", "rust_test") + +def _sharding_enabled_test(ctx): + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + executable = tut[DefaultInfo].files_to_run.executable + + asserts.true( + env, + executable.basename.endswith("_sharding_wrapper.sh") or + executable.basename.endswith("_sharding_wrapper.bat"), + "Expected sharding wrapper script, got: " + executable.basename, + ) + + return analysistest.end(env) + +sharding_enabled_test = analysistest.make(_sharding_enabled_test) + +def _sharding_disabled_test(ctx): + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + executable = tut[DefaultInfo].files_to_run.executable + + asserts.false( + env, + executable.basename.endswith("_sharding_wrapper.sh") or + executable.basename.endswith("_sharding_wrapper.bat"), + "Expected test binary, not wrapper script: " + executable.basename, + ) + + return analysistest.end(env) + +sharding_disabled_test = analysistest.make(_sharding_disabled_test) + +def _test_sharding_targets(): + rust_test( + name = "sharded_test_enabled", + srcs = ["sharded_test.rs"], + edition = "2021", + experimental_enable_sharding = True, + ) + + sharding_enabled_test( + name = "sharding_enabled_test", + target_under_test = ":sharded_test_enabled", + ) + + rust_test( + name = "sharded_test_disabled", + srcs = ["sharded_test.rs"], + edition = "2021", + experimental_enable_sharding = False, + ) + + sharding_disabled_test( + name = "sharding_disabled_test", + target_under_test = ":sharded_test_disabled", + ) + + rust_test( + name = "sharded_integration_test", + srcs = ["sharded_test.rs"], + edition = "2021", + experimental_enable_sharding = True, + shard_count = 3, + ) + + sh_test( + name = "test_sharding_wrapper_hashes_sorted_names", + srcs = ["test_sharding_wrapper_hashes_sorted_names.sh"], + args = [ + "$(location //rust/private:test_sharding_wrapper.sh)", + "$(location :fake_libtest_binary.sh)", + ], + data = [ + ":fake_libtest_binary.sh", + "//rust/private:test_sharding_wrapper.sh", + ], + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + ) + +def test_sharding_test_suite(name): + _test_sharding_targets() + + native.test_suite( + name = name, + tests = [ + ":sharding_enabled_test", + ":sharding_disabled_test", + ":test_sharding_wrapper_hashes_sorted_names", + ":sharded_integration_test", + ], + ) diff --git a/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh new file mode 100755 index 0000000000..626f33da44 --- /dev/null +++ b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euo pipefail + +wrapper_template=$1 +fake_binary_src=$2 + +workdir="${TEST_TMPDIR:-$(mktemp -d)}" +fake_binary="$workdir/fake_libtest_binary" +wrapper="$workdir/wrapper.sh" + +cp "$fake_binary_src" "$fake_binary" +chmod +x "$fake_binary" + +sed 's|{{TEST_BINARY}}|fake_libtest_binary|g' "$wrapper_template" > "$wrapper" +chmod +x "$wrapper" + +collect_mapping() { + local variant=$1 + local order=$2 + local output=$3 + local unsorted_output="${output}.unsorted" + local shard + + : > "$unsorted_output" + for shard in 0 1 2; do + local shard_output="$workdir/${variant}_${order}_${shard}.txt" + : > "$shard_output" + + ( + cd "$workdir" + TEST_LIST_VARIANT="$variant" \ + TEST_LIST_ORDER="$order" \ + TEST_SHARD_OUTPUT="$shard_output" \ + RULES_RUST_TEST_SHARD_INDEX="$shard" \ + RULES_RUST_TEST_TOTAL_SHARDS=3 \ + ./wrapper.sh + ) + + while IFS= read -r test_name; do + printf '%s %s\n' "$test_name" "$shard" >> "$unsorted_output" + done < "$shard_output" + done + + LC_ALL=C sort "$unsorted_output" > "$output" +} + +assert_same_mapping() { + local expected=$1 + local actual=$2 + local message=$3 + + if ! diff -u "$expected" "$actual"; then + echo "$message" >&2 + exit 1 + fi +} + +base_normal="$workdir/base_normal.txt" +base_reversed="$workdir/base_reversed.txt" +with_added="$workdir/with_added.txt" +with_added_existing_tests="$workdir/with_added_existing_tests.txt" + +collect_mapping base normal "$base_normal" +collect_mapping base reversed "$base_reversed" +collect_mapping with_added normal "$with_added" + +assert_same_mapping "$base_normal" "$base_reversed" \ + "test shard assignment changed when libtest list order changed" + +sed '/^aardvark::test_added /d' "$with_added" > "$with_added_existing_tests" +assert_same_mapping "$base_normal" "$with_added_existing_tests" \ + "existing test shard assignment changed after adding a new test" From 2e8ce7341fb147fb98e0bdd344dd04fa433a7a1e Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 26 Apr 2026 13:17:02 -0400 Subject: [PATCH 23/50] Add per-crate rustc flag trimming --- rust/private/per_crate_flag_trim.bzl | 61 ++++++++++++++++++++++++++++ rust/private/rust.bzl | 43 ++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 rust/private/per_crate_flag_trim.bzl diff --git a/rust/private/per_crate_flag_trim.bzl b/rust/private/per_crate_flag_trim.bzl new file mode 100644 index 0000000000..d0e76684a5 --- /dev/null +++ b/rust/private/per_crate_flag_trim.bzl @@ -0,0 +1,61 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transition to trim per_crate_rustc_flag for non-matching targets. + +This module provides a configuration trimming mechanism for the +`experimental_per_crate_rustc_flag` setting. When a target has +`skip_per_crate_rustc_flags = True`, this transition clears the setting, +putting the target back into a canonical configuration. + +This is useful for third-party crates (e.g., from crate_universe) that will +never match any per-crate flag filter. Without trimming, these crates would +be rebuilt unnecessarily when any per-crate flag is set, even though the +filter doesn't match them. + +Usage: + Third-party crate generators (like crate_universe) should set + `skip_per_crate_rustc_flags = True` on generated rust_library targets. +""" + +_PER_CRATE_FLAG_SETTING = "@rules_rust//rust/settings:experimental_per_crate_rustc_flag" + +def _per_crate_flag_trim_transition_impl(settings, attr): + """Clear per_crate_rustc_flag for targets marked to skip it. + + Args: + settings: A dict of current build settings. + attr: The attributes of the target being configured. + + Returns: + A dict with the per_crate_rustc_flag setting (cleared or preserved). + """ + + # If this target is marked to skip per-crate flags, clear the setting + # to return it to a canonical configuration + if getattr(attr, "skip_per_crate_rustc_flags", False): + return { + _PER_CRATE_FLAG_SETTING: [], + } + + # Otherwise, keep the current value + return { + _PER_CRATE_FLAG_SETTING: settings[_PER_CRATE_FLAG_SETTING], + } + +per_crate_flag_trim_transition = transition( + implementation = _per_crate_flag_trim_transition_impl, + inputs = [_PER_CRATE_FLAG_SETTING], + outputs = [_PER_CRATE_FLAG_SETTING], +) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 75733e7d0d..c680a732cc 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -18,6 +18,10 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":common.bzl", "COMMON_PROVIDERS", "rust_common") +load( + ":per_crate_flag_trim.bzl", + "per_crate_flag_trim_transition", +) load( ":providers.bzl", "BuildInfo", @@ -59,6 +63,9 @@ load( # TODO(marco): Separate each rule into its own file. +# Setting path for per_crate_rustc_flag, used in transition definitions +_PER_CRATE_FLAG_SETTING = "@rules_rust//rust/settings:experimental_per_crate_rustc_flag" + def _toolchain_make_variables(ctx): make_variables = {} for toolchain_target in getattr(ctx.attr, "toolchains", []): @@ -867,6 +874,17 @@ _COMMON_ATTRS = { doc = "A setting used to determine whether or not the `--stamp` flag is enabled", default = Label("//rust/private:stamp"), ), + "skip_per_crate_rustc_flags": attr.bool( + doc = dedent("""\ + If True, the `experimental_per_crate_rustc_flag` setting is trimmed from this + target's configuration. This puts the target back into a canonical configuration, + improving cache hit rates for targets that would never match any per-crate filter. + + This attribute is primarily used by crate_universe for generated third-party crates. + First-party crates should leave this as False (the default). + """), + default = False, + ), } | RUSTC_ATTRS | RUSTC_ALLOCATOR_LIBRARIES_ATTRS _PLATFORM_ATTRS = { @@ -997,6 +1015,7 @@ _RUST_TEST_ATTRS = { rust_library = rule( implementation = _rust_library_impl, provides = COMMON_PROVIDERS, + cfg = per_crate_flag_trim_transition, attrs = _COMMON_ATTRS | { "disable_pipelining": attr.bool( default = False, @@ -1006,6 +1025,9 @@ rust_library = rule( crates will instead use the `.rlib` file. """), ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), }, fragments = ["cpp"], toolchains = [ @@ -1099,17 +1121,22 @@ def _resolve_platform(settings, attr): return platform def _rust_static_library_transition_impl(settings, attr): + # Trim per_crate_rustc_flag for third-party crates to improve cache hit rates + per_crate_flags = [] if getattr(attr, "skip_per_crate_rustc_flags", False) else settings[_PER_CRATE_FLAG_SETTING] return { "//command_line_option:platforms": _resolve_platform(settings, attr), + _PER_CRATE_FLAG_SETTING: per_crate_flags, } _rust_static_library_transition = transition( implementation = _rust_static_library_transition_impl, inputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], outputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], ) @@ -1140,17 +1167,22 @@ rust_static_library = rule( ) def _rust_shared_library_transition_impl(settings, attr): + # Trim per_crate_rustc_flag for third-party crates to improve cache hit rates + per_crate_flags = [] if getattr(attr, "skip_per_crate_rustc_flags", False) else settings[_PER_CRATE_FLAG_SETTING] return { "//command_line_option:platforms": _resolve_platform(settings, attr), + _PER_CRATE_FLAG_SETTING: per_crate_flags, } _rust_shared_library_transition = transition( implementation = _rust_shared_library_transition_impl, inputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], outputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], ) @@ -1195,6 +1227,7 @@ _proc_macro_dep_transition = transition( rust_proc_macro = rule( implementation = _rust_proc_macro_impl, provides = COMMON_PROVIDERS, + cfg = per_crate_flag_trim_transition, # Start by copying the common attributes, then override the `deps` attribute # to apply `_proc_macro_dep_transition`. To add this transition we additionally # need to declare `_allowlist_function_transition`, see @@ -1262,17 +1295,22 @@ _RUST_BINARY_ATTRS = { } | _EXPERIMENTAL_USE_CC_COMMON_LINK_ATTRS def _rust_binary_transition_impl(settings, attr): + # Trim per_crate_rustc_flag for third-party crates to improve cache hit rates + per_crate_flags = [] if getattr(attr, "skip_per_crate_rustc_flags", False) else settings[_PER_CRATE_FLAG_SETTING] return { "//command_line_option:platforms": _resolve_platform(settings, attr), + _PER_CRATE_FLAG_SETTING: per_crate_flags, } _rust_binary_transition = transition( implementation = _rust_binary_transition_impl, inputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], outputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], ) @@ -1511,17 +1549,22 @@ rust_test_without_process_wrapper_test = rule( ) def _rust_test_transition_impl(settings, attr): + # Trim per_crate_rustc_flag for third-party crates to improve cache hit rates + per_crate_flags = [] if getattr(attr, "skip_per_crate_rustc_flags", False) else settings[_PER_CRATE_FLAG_SETTING] return { "//command_line_option:platforms": _resolve_platform(settings, attr), + _PER_CRATE_FLAG_SETTING: per_crate_flags, } _rust_test_transition = transition( implementation = _rust_test_transition_impl, inputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], outputs = [ "//command_line_option:platforms", + _PER_CRATE_FLAG_SETTING, ], ) From 65a2ba769c357442f2f6a5d3cb324cc9fead47ec Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 26 Apr 2026 13:18:48 -0400 Subject: [PATCH 24/50] Fix Windows GNU staticlib output naming Rustc emits GNU-like Windows staticlibs as lib.a, but rules_rust was stripping the lib prefix for all Windows non-rlib library outputs. Keep the prefix for staticlib outputs when the target ABI is gnu or gnullvm so declared outputs match rustc. --- rust/private/utils.bzl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index c84610b126..810ea0b93a 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -865,7 +865,12 @@ def determine_lib_name(name, crate_type, toolchain, lib_hash = None): "please file an issue!").format(crate_type)) prefix = "lib" - if toolchain.target_triple and toolchain.target_os == "windows" and crate_type not in ("lib", "rlib"): + if (toolchain.target_triple and + toolchain.target_os == "windows" and + crate_type not in ("lib", "rlib") and + # GNU-like Windows staticlibs are archives named lib.a. + (crate_type != "staticlib" or + toolchain.target_abi not in ("gnu", "gnullvm"))): prefix = "" if toolchain.target_arch in ("wasm32", "wasm64") and crate_type == "cdylib": prefix = "" From 57ff613b81e938bbd2804626318fb67b4aa92aa7 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 19 Dec 2025 08:29:48 -0500 Subject: [PATCH 25/50] Move processwrapper to rust toolchain --- .../rust/aarch64-apple-darwin/BUILD.bazel | 1 + .../rust/aarch64-apple-ios/BUILD.bazel | 1 + .../rust/aarch64-linux-android/BUILD.bazel | 1 + .../aarch64-unknown-linux-gnu/BUILD.bazel | 1 + .../rust/wasm32-unknown-unknown/BUILD.bazel | 1 + .../toolchains/rust/wasm32-wasip1/BUILD.bazel | 1 + .../rust/x86_64-apple-darwin/BUILD.bazel | 1 + .../rust/x86_64-pc-windows-msvc/BUILD.bazel | 1 + .../rust/x86_64-unknown-linux-gnu/BUILD.bazel | 1 + .../rust/x86_64-unknown-nixos-gnu/BUILD.bazel | 1 + extensions/bindgen/private/bindgen.bzl | 6 --- extensions/prost/private/prost.bzl | 2 +- .../private/wasm_bindgen_test.bzl | 2 +- rust/private/BUILD.bazel | 24 +++++++++ rust/private/clippy.bzl | 12 +---- rust/private/repository_utils.bzl | 52 +++++++++++++++++- rust/private/rust.bzl | 54 ++++++++----------- rust/private/rustc.bzl | 18 ++++--- rust/private/rustdoc.bzl | 9 +--- rust/private/rustdoc_test.bzl | 10 +--- rust/private/rustfmt.bzl | 10 ++-- rust/private/unpretty.bzl | 2 +- rust/private/utils.bzl | 7 +-- rust/toolchain.bzl | 19 +++++++ .../unit/cc_common_link_test.bzl | 1 + test/toolchain/toolchain_test.bzl | 2 + .../with_modified_crate_name.bzl | 6 --- test/unit/force_all_deps_direct/generator.bzl | 6 --- test/unit/pipelined_compilation/wrap.bzl | 6 --- test/unit/toolchain/toolchain_test.bzl | 4 ++ util/process_wrapper/BUILD.bazel | 4 ++ 31 files changed, 160 insertions(+), 106 deletions(-) diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-darwin/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-darwin/BUILD.bazel index 345779edb9..77a6906415 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-darwin/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-darwin/BUILD.bazel @@ -18,6 +18,7 @@ rust_toolchain( extra_rustc_flags = [ "-Clinker-flavor=ld64.lld", ], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-aarch64-apple-darwin", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-ios/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-ios/BUILD.bazel index da57cd78c8..cb3ea66738 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-ios/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-apple-ios/BUILD.bazel @@ -18,6 +18,7 @@ rust_toolchain( extra_rustc_flags = [ "-Clinker-flavor=ld64.lld", ], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-aarch64-apple-ios", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-linux-android/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-linux-android/BUILD.bazel index 3c56b48333..fc3058535b 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-linux-android/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-linux-android/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-aarch64-linux-android", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-unknown-linux-gnu/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-unknown-linux-gnu/BUILD.bazel index 215b7d74d1..fc2f8560ba 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-unknown-linux-gnu/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/aarch64-unknown-linux-gnu/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-aarch64-unknown-linux-gnu", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-unknown-unknown/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-unknown-unknown/BUILD.bazel index 54fcb388c7..a48d1b2231 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-unknown-unknown/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-unknown-unknown/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-wasm32-unknown-unknown", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-wasip1/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-wasip1/BUILD.bazel index 19aa94c23e..f5c71409e2 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-wasip1/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/wasm32-wasip1/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-wasm32-wasip1", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-apple-darwin/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-apple-darwin/BUILD.bazel index 9ad447eefa..d4bba2b7c3 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-apple-darwin/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-apple-darwin/BUILD.bazel @@ -18,6 +18,7 @@ rust_toolchain( extra_rustc_flags = [ "-Clinker-flavor=ld64.lld", ], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-x86_64-apple-darwin", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-pc-windows-msvc/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-pc-windows-msvc/BUILD.bazel index d03d580879..1b9050b5ec 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-pc-windows-msvc/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-pc-windows-msvc/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-x86_64-pc-windows-msvc", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-linux-gnu/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-linux-gnu/BUILD.bazel index 3a79e39705..c73aedad4f 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-linux-gnu/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-linux-gnu/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-x86_64-unknown-linux-gnu", rustc = "@nix_rust//:rustc", diff --git a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-nixos-gnu/BUILD.bazel b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-nixos-gnu/BUILD.bazel index d237d28fbc..c877356072 100644 --- a/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-nixos-gnu/BUILD.bazel +++ b/examples/cross_compile_nix/bazel/toolchains/rust/x86_64-unknown-nixos-gnu/BUILD.bazel @@ -11,6 +11,7 @@ rust_toolchain( exec_triple = "x86_64-unknown-nixos-gnu", extra_exec_rustc_flags = [], extra_rustc_flags = [], + process_wrapper = "@rules_rust//util/process_wrapper", rust_doc = "@nix_rust//:rustdoc", rust_std = "@nix_rust//:rust_std-x86_64-unknown-linux-gnu", rustc = "@nix_rust//:rustc", diff --git a/extensions/bindgen/private/bindgen.bzl b/extensions/bindgen/private/bindgen.bzl index c436c4ea31..a25a0bc19e 100644 --- a/extensions/bindgen/private/bindgen.bzl +++ b/extensions/bindgen/private/bindgen.bzl @@ -472,12 +472,6 @@ rust_bindgen = rule( doc = "Whether to create a separate .c file for static fns. Requires nightly toolchain, and a header that actually needs this feature (otherwise bindgen won't generate the file and Bazel complains).", default = False, ), - "_process_wrapper": attr.label( - default = Label("@rules_rust//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), }, outputs = {"out": "%{name}.rs"}, fragments = ["cpp"], diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index e67ff89fcf..37d9a7ee81 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -196,7 +196,7 @@ def _compile_rust( extension = "_meta.rlib", ) rmeta = ctx.actions.declare_file(rmeta_name) - rustc_rmeta_output = generate_output_diagnostics(ctx, rmeta) + rustc_rmeta_output = generate_output_diagnostics(ctx, toolchain, rmeta) metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, "rlib") providers = rustc_compile_action( diff --git a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl index 6c26f6c3cf..9d8a18e6b8 100644 --- a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl +++ b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl @@ -119,7 +119,7 @@ def _rust_wasm_bindgen_test_binary_impl(ctx): proc_macro_deps = depset(proc_macro_deps, transitive = [crate.proc_macro_deps]).to_list(), aliases = {}, output = output, - rustc_output = generate_output_diagnostics(ctx, output), + rustc_output = generate_output_diagnostics(ctx, toolchain, output), edition = crate.edition, rustc_env = rustc_env, rustc_env_files = rustc_env_files, diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index 4a9afc2995..7d8fa2cbcd 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -1,4 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_setting") load("//rust/private:rust_analyzer.bzl", "rust_analyzer_detect_sysroot") load("//rust/private:rustc.bzl", "is_proc_macro_dep", "is_proc_macro_dep_enabled") load("//rust/private:stamp.bzl", "stamp_build_setting") @@ -62,3 +63,26 @@ rust_analyzer_detect_sysroot( name = "rust_analyzer_detect_sysroot", visibility = ["//visibility:public"], ) + +# This setting lets us configure a bootstrap toolchain to build the process_wrapper +# and a "full" toolchain (that uses process_wrapper) to build user code. +bool_setting( + name = "bootstrap_setting", + build_setting_default = False, +) + +config_setting( + name = "bootstrapped", + flag_values = { + ":bootstrap_setting": "0", + }, + visibility = ["//visibility:public"], +) + +config_setting( + name = "bootstrapping", + flag_values = { + ":bootstrap_setting": "1", + }, + visibility = ["//visibility:public"], +) diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index 3f54e4162e..cdc7e25cd8 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -101,13 +101,12 @@ def get_clippy_ready_crate_info(target, aspect_ctx = None): else: return None -def rust_clippy_action(ctx, clippy_executable, process_wrapper, crate_info, config, output = None, success_marker = None, cap_at_warnings = False, extra_clippy_flags = [], error_format = None, clippy_diagnostics_file = None): +def rust_clippy_action(ctx, clippy_executable, crate_info, config, output = None, success_marker = None, cap_at_warnings = False, extra_clippy_flags = [], error_format = None, clippy_diagnostics_file = None): """Run clippy with the specified parameters. Args: ctx (ctx): The aspect's context object. This function should not read ctx.attr, but it might read ctx.rule.attr clippy_executable (File): The clippy executable to run - process_wrapper (File): An executable process wrapper that can run clippy, usually @rules_rust//utils/process_wrapper crate_info (CrateInfo): The source crate information config (File): The clippy configuration file. Reference: https://doc.rust-lang.org/clippy/configuration.html#configuring-clippy output (File): The output file for clippy stdout/stderr. If None, no output will be captured @@ -224,7 +223,7 @@ def rust_clippy_action(ctx, clippy_executable, process_wrapper, crate_info, conf compile_inputs = depset([config], transitive = [compile_inputs]) ctx.actions.run( - executable = process_wrapper, + executable = toolchain.process_wrapper, inputs = compile_inputs, outputs = outputs + [x for x in [clippy_diagnostics_file] if x], env = env, @@ -279,7 +278,6 @@ def _clippy_aspect_impl(target, ctx): rust_clippy_action( ctx = ctx, clippy_executable = toolchain.clippy_driver, - process_wrapper = ctx.executable._process_wrapper, crate_info = crate_info, config = ctx.file._config, output = clippy_out, @@ -347,12 +345,6 @@ rust_clippy_aspect = aspect( "_per_crate_rustc_flag": attr.label( default = Label("//rust/settings:experimental_per_crate_rustc_flag"), ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running clippy on all platforms", - default = Label("//util/process_wrapper"), - executable = True, - cfg = "exec", - ), }, provides = [ClippyInfo], required_providers = [ diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index c1084bf60d..be0622db95 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -343,6 +343,41 @@ rust_toolchain( rust_doc = "//:rustdoc", rust_std = "//:rust_std-{target_triple}", rustc = "//:rustc", + process_wrapper = "@rules_rust//util/process_wrapper", + linker = {linker_label}, + linker_type = {linker_type}, + rustfmt = {rustfmt_label}, + cargo = "//:cargo", + clippy_driver = "//:clippy_driver_bin", + cargo_clippy = "//:cargo_clippy_bin", + llvm_cov = {llvm_cov_label}, + llvm_profdata = {llvm_profdata_label}, + llvm_lib = {llvm_lib_label}, + rustc_lib = "//:rustc_lib", + allocator_library = {allocator_library}, + global_allocator_library = {global_allocator_library}, + binary_ext = "{binary_ext}", + staticlib_ext = "{staticlib_ext}", + dylib_ext = "{dylib_ext}", + stdlib_linkflags = [{stdlib_linkflags}], + default_edition = "{default_edition}", + exec_triple = "{exec_triple}", + target_triple = "{target_triple}", + visibility = ["//visibility:public"], + extra_rustc_flags = {extra_rustc_flags}, + extra_exec_rustc_flags = {extra_exec_rustc_flags}, + opt_level = {opt_level}, + strip_level = {strip_level}, + tags = ["rust_version={version}"], +) + +rust_toolchain( + name = "{toolchain_name}_bootstrap", + bootstrapping = True, + rust_doc = "//:rustdoc", + rust_std = "//:rust_std-{target_triple}", + rustc = "//:rustc", + process_wrapper = "@rules_rust//util/process_wrapper:bootstrap_process_wrapper", linker = {linker_label}, linker_type = {linker_type}, rust_objcopy = {rust_objcopy_label}, @@ -493,7 +528,20 @@ toolchain( target_compatible_with = {target_constraint_sets_serialized}, toolchain = "{toolchain}", toolchain_type = "{toolchain_type}", - {target_settings} + target_settings = [ + "@rules_rust//rust/private:bootstrapped",{target_settings} + ], +) + +toolchain( + name = "{name}_bootstrap", + exec_compatible_with = {exec_constraint_sets_serialized}, + target_compatible_with = {target_constraint_sets_serialized}, + toolchain = "{toolchain}_bootstrap", + toolchain_type = "{toolchain_type}", + target_settings = [ + "@rules_rust//rust/private:bootstrapping",{target_settings} + ], ) """ @@ -504,7 +552,7 @@ def BUILD_for_toolchain( target_settings, target_compatible_with, exec_compatible_with): - target_settings_value = "target_settings = {},".format(json.encode(target_settings)) if target_settings else "# target_settings = []" + target_settings_value = ",\n ".join([repr(setting) for setting in target_settings]) return _build_file_for_toolchain_template.format( name = name, diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index c680a732cc..7a5817b774 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -192,7 +192,7 @@ def _rust_library_common(ctx, crate_type): paths.replace_extension(rust_lib_name, "_meta.rlib"), sibling = rust_lib, ) - rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + rustc_rmeta_output = generate_output_diagnostics(ctx, toolchain, rust_metadata) metadata_supports_pipelining = ( can_use_metadata_for_pipelining(toolchain, crate_type) and not ctx.attr.disable_pipelining @@ -215,7 +215,7 @@ def _rust_library_common(ctx, crate_type): proc_macro_deps = proc_macro_deps, aliases = ctx.attr.aliases, output = rust_lib, - rustc_output = generate_output_diagnostics(ctx, rust_lib), + rustc_output = generate_output_diagnostics(ctx, toolchain, rust_lib), metadata = rust_metadata, metadata_supports_pipelining = metadata_supports_pipelining, rustc_rmeta_output = rustc_rmeta_output, @@ -265,7 +265,7 @@ def _rust_binary_impl(ctx): paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) - rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + rustc_rmeta_output = generate_output_diagnostics(ctx, toolchain, rust_metadata) providers = rustc_compile_action( ctx = ctx, @@ -280,7 +280,7 @@ def _rust_binary_impl(ctx): proc_macro_deps = proc_macro_deps, aliases = ctx.attr.aliases, output = output, - rustc_output = generate_output_diagnostics(ctx, output), + rustc_output = generate_output_diagnostics(ctx, toolchain, output), metadata = rust_metadata, rustc_rmeta_output = rustc_rmeta_output, edition = get_edition(ctx.attr, toolchain, ctx.label), @@ -367,7 +367,7 @@ def _rust_test_impl(ctx): paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) - rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + rustc_rmeta_output = generate_output_diagnostics(ctx, toolchain, rust_metadata) # Need to consider all src files together when transforming srcs = depset(ctx.files.srcs, transitive = [crate.srcs]).to_list() @@ -402,7 +402,7 @@ def _rust_test_impl(ctx): proc_macro_deps = depset(proc_macro_deps, transitive = [crate.proc_macro_deps]).to_list(), aliases = aliases, output = output, - rustc_output = generate_output_diagnostics(ctx, output), + rustc_output = generate_output_diagnostics(ctx, toolchain, output), metadata = rust_metadata, rustc_rmeta_output = rustc_rmeta_output, edition = crate.edition, @@ -437,7 +437,7 @@ def _rust_test_impl(ctx): paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) - rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + rustc_rmeta_output = generate_output_diagnostics(ctx, toolchain, rust_metadata) if ctx.attr.rustc_env: rustc_env = expand_dict_value_locations( @@ -459,7 +459,7 @@ def _rust_test_impl(ctx): proc_macro_deps = proc_macro_deps, aliases = ctx.attr.aliases, output = output, - rustc_output = generate_output_diagnostics(ctx, output), + rustc_output = generate_output_diagnostics(ctx, toolchain, output), metadata = rust_metadata, rustc_rmeta_output = rustc_rmeta_output, edition = get_edition(ctx.attr, toolchain, ctx.label), @@ -682,13 +682,6 @@ RUSTC_ATTRS = { "_per_crate_rustc_flag": attr.label( default = Label("//rust/settings:experimental_per_crate_rustc_flag"), ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running rustc on all platforms.", - default = Label("//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), "_rustc_output_diagnostics": attr.label( default = Label("//rust/settings:rustc_output_diagnostics"), ), @@ -1418,21 +1411,6 @@ rust_binary = rule( def _common_attrs_for_binary_without_process_wrapper(attrs): new_attr = dict(attrs) - # use a fake process wrapper - new_attr["_process_wrapper"] = attr.label( - default = None, - executable = True, - allow_single_file = True, - cfg = "exec", - ) - - new_attr["_bootstrap_process_wrapper"] = attr.label( - default = Label("//util/process_wrapper:bootstrap_process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ) - # fix stamp = 0 new_attr["stamp"] = attr.int( doc = dedent("""\ @@ -1450,22 +1428,32 @@ _RustBuiltWithoutProcessWrapperInfo = provider( fields = {}, ) +def _bootstrap_process_wrapper_transition_impl(_settings, _attr): + return {str(Label("//rust/private:bootstrap_setting")): True} + +_bootstrap_process_wrapper_transition = transition( + implementation = _bootstrap_process_wrapper_transition_impl, + inputs = [], + outputs = [str(Label("//rust/private:bootstrap_setting"))], +) + def _rust_binary_without_process_wrapper_impl(ctx): providers = _rust_binary_impl(ctx) return providers + [_RustBuiltWithoutProcessWrapperInfo()] -# Provides an internal rust_{binary,library} to use that we can use to build the process -# wrapper, this breaks the dependency of rust_* on the process wrapper by -# setting it to None, which the functions in rustc detect and build accordingly. rust_binary_without_process_wrapper = rule( implementation = _rust_binary_without_process_wrapper_impl, doc = "A variant of `rust_binary` that uses a minimal process wrapper for `Rustc` actions.", provides = COMMON_PROVIDERS + [_RustBuiltWithoutProcessWrapperInfo], attrs = _common_attrs_for_binary_without_process_wrapper(_COMMON_ATTRS | _RUST_BINARY_ATTRS | { + "_allowlist_function_transition": attr.label( + default = Label("//tools/allowlists/function_transition_allowlist"), + ), "_skip_deps_verification": attr.bool(default = True), }), executable = True, fragments = ["cpp"], + cfg = _bootstrap_process_wrapper_transition, toolchains = [ str(Label("//rust:toolchain_type")), config_common.toolchain_type("@bazel_tools//tools/cpp:toolchain_type", mandatory = False), diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 0fc3a78a7a..b87ef5d4c0 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1615,7 +1615,7 @@ def rustc_compile_action( use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, - always_use_param_file = not ctx.executable._process_wrapper, + always_use_param_file = toolchain._bootstrapping, allowed_unstable_rust_features = allowed_unstable_rust_features, ) @@ -1707,10 +1707,14 @@ def rustc_compile_action( dsym_folder = ctx.actions.declare_directory(crate_info.output.basename + ".dSYM", sibling = crate_info.output) action_outputs.append(dsym_folder) - if ctx.executable._process_wrapper: + process_wrapper = toolchain.process_wrapper + if not process_wrapper: + fail("No process wrapper was defined for {}".format(ctx.label)) + + if not toolchain._bootstrapping: # Run as normal ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = process_wrapper, inputs = compile_inputs, outputs = action_outputs, env = env, @@ -1728,7 +1732,7 @@ def rustc_compile_action( ) if args_metadata: ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = process_wrapper, inputs = compile_inputs_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, @@ -1743,12 +1747,12 @@ def rustc_compile_action( toolchain = "@rules_rust//rust:toolchain_type", execution_requirements = {"supports-path-mapping": ""} if args_metadata.supports_path_mapping else None, ) - elif hasattr(ctx.executable, "_bootstrap_process_wrapper"): + else: # Run without process_wrapper if build_env_files or build_flags_files or stamp or build_metadata: fail("build_env_files, build_flags_files, stamp, build_metadata are not supported when building without process_wrapper") ctx.actions.run( - executable = ctx.executable._bootstrap_process_wrapper, + executable = process_wrapper, inputs = compile_inputs, outputs = action_outputs, env = env, @@ -1764,8 +1768,6 @@ def rustc_compile_action( resource_set = get_rustc_resource_set(toolchain), execution_requirements = {"supports-path-mapping": ""} if args.supports_path_mapping else None, ) - else: - fail("No process wrapper was defined for {}".format(ctx.label)) if experimental_use_cc_common_link: # Wrap the main `.o` file into a compilation output suitable for diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index f302bab743..074b7fec46 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -164,7 +164,7 @@ def rustdoc_compile_action( all_inputs = depset([crate_info.output], transitive = [compile_inputs, depset(html_input_files)]) return struct( - executable = ctx.executable._process_wrapper, + executable = toolchain.process_wrapper, inputs = all_inputs, env = env, arguments = args.all, @@ -365,13 +365,6 @@ rust_doc = rule( "_error_format": attr.label( default = Label("//rust/settings:error_format"), ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running rustdoc on all platforms", - default = Label("@rules_rust//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), "_zipper": attr.label( doc = "A Bazel provided tool for creating archives", default = Label("@bazel_tools//tools/zip:zipper"), diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index dcf847f10a..d521ccd279 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -93,7 +93,7 @@ def _construct_writer_arguments(ctx, test_runner, opt_test_params, action, crate # Prepare for the process runner to ingest the rest of the arguments # to match the expectations of `rustc_compile_action`. - writer_args.add(ctx.executable._process_wrapper.short_path) + writer_args.add(action.executable.short_path) return (writer_args, action.env) @@ -162,7 +162,7 @@ def _rust_doc_test_impl(ctx): is_test = True, ) - tools = action.tools + [ctx.executable._process_wrapper] + tools = action.tools + [action.executable] writer_args, env = _construct_writer_arguments( ctx = ctx, @@ -235,12 +235,6 @@ rust_doc_test = rule( file of arguments to rustc: `@$(location //package:target)`. """), ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running rustdoc on all platforms", - cfg = "exec", - default = Label("//util/process_wrapper"), - executable = True, - ), "_test_writer": attr.label( doc = "A binary used for writing script for use as the test executable.", cfg = "exec", diff --git a/rust/private/rustfmt.bzl b/rust/private/rustfmt.bzl index efa17efaa8..cff31edb0b 100644 --- a/rust/private/rustfmt.bzl +++ b/rust/private/rustfmt.bzl @@ -75,6 +75,7 @@ def _generate_manifest(edition, srcs, ctx): return manifest def _perform_check(edition, srcs, ctx): + rust_toolchain = ctx.toolchains[Label("//rust:toolchain_type")] rustfmt_toolchain = ctx.toolchains[Label("//rust/rustfmt:toolchain_type")] config = ctx.file._config @@ -90,7 +91,7 @@ def _perform_check(edition, srcs, ctx): args.add_all(srcs) ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = rust_toolchain.process_wrapper, inputs = srcs + [config], outputs = [marker], tools = [rustfmt_toolchain.all_files], @@ -181,12 +182,6 @@ generated source files are also ignored by this aspect. allow_single_file = True, default = Label("//rust/settings:rustfmt.toml"), ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running rustfmt on all platforms", - cfg = "exec", - executable = True, - default = Label("//util/process_wrapper"), - ), }, required_providers = [ [rust_common.crate_info], @@ -195,6 +190,7 @@ generated source files are also ignored by this aspect. requires = [rustfmt_srcs_aspect], fragments = ["cpp"], toolchains = [ + str(Label("//rust:toolchain_type")), str(Label("//rust/rustfmt:toolchain_type")), ], ) diff --git a/rust/private/unpretty.bzl b/rust/private/unpretty.bzl index b8ba474fd5..d82fe708be 100644 --- a/rust/private/unpretty.bzl +++ b/rust/private/unpretty.bzl @@ -214,7 +214,7 @@ def _rust_unpretty_aspect_impl(target, ctx): args.rustc_flags.add("-Zunpretty={}".format(mode)) ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = toolchain.process_wrapper, inputs = compile_inputs, outputs = [unpretty_out], env = env, diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index 810ea0b93a..6ee717882e 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -759,7 +759,7 @@ def can_build_metadata(toolchain, ctx, crate_type, *, disable_pipelining = False # 2) either: # * always_enable_metadata_output_groups is set # * this target can use metadata for pipelined compilation - return bool(ctx.attr._process_wrapper) and ( + return not toolchain._bootstrapping and ( ctx.attr._always_enable_metadata_output_groups[AlwaysEnableMetadataOutputGroupsInfo].always_enable_metadata_output_groups or (not disable_pipelining and can_use_metadata_for_pipelining(toolchain, crate_type)) @@ -971,11 +971,12 @@ def _symlink_for_non_generated_source(ctx, src_file, package_root): else: return src_file -def generate_output_diagnostics(ctx, sibling, require_process_wrapper = True): +def generate_output_diagnostics(ctx, toolchain, sibling, require_process_wrapper = True): """Generates a .rustc-output file if it's required. Args: ctx: (ctx): The current rule's context object + toolchain: (Rust toolchain): The current rust toolchain sibling: (File): The file to generate the diagnostics for. require_process_wrapper: (bool): Whether to require the process wrapper in order to generate the .rustc-output file. @@ -986,7 +987,7 @@ def generate_output_diagnostics(ctx, sibling, require_process_wrapper = True): # Since this feature requires error_format=json, we usually need # process_wrapper, since it can write the json here, then convert it to the # regular error format so the user can see the error properly. - if require_process_wrapper and not ctx.attr._process_wrapper: + if require_process_wrapper and toolchain._bootstrapping: return None provider = ctx.attr._rustc_output_diagnostics[RustcOutputDiagnosticsInfo] if not provider.rustc_output_diagnostics: diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index c62263daf7..698b600fb7 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -412,6 +412,11 @@ def _rust_toolchain_impl(ctx): no_std = ctx.attr._no_std[BuildSettingInfo].value lto = ctx.attr.lto[RustLtoInfo] + if not ctx.attr.process_wrapper: + fail("rust_toolchain.process_wrapper must be set. Please update {}".format( + ctx.label, + )) + experimental_use_global_allocator = ctx.attr._experimental_use_global_allocator[BuildSettingInfo].value if _experimental_use_cc_common_link(ctx): if experimental_use_global_allocator and not ctx.attr.global_allocator_library: @@ -622,6 +627,7 @@ def _rust_toolchain_impl(ctx): extra_rustc_flags = expanded_extra_rustc_flags, extra_rustc_flags_for_crate_types = ctx.attr.extra_rustc_flags_for_crate_types, extra_exec_rustc_flags = expanded_extra_exec_rustc_flags, + process_wrapper = ctx.executable.process_wrapper if ctx.attr.process_wrapper else None, per_crate_rustc_flags = ctx.attr.per_crate_rustc_flags, sysroot = sysroot_path, sysroot_anchor = sysroot.sysroot_anchor, @@ -647,6 +653,7 @@ def _rust_toolchain_impl(ctx): _incompatible_do_not_include_data_in_compile_data = ctx.attr._incompatible_do_not_include_data_in_compile_data[IncompatibleFlagInfo].enabled, _incompatible_do_not_include_transitive_data_in_compile_inputs = ctx.attr._incompatible_do_not_include_transitive_data_in_compile_inputs[IncompatibleFlagInfo].enabled, _no_std = no_std, + _bootstrapping = ctx.attr.bootstrapping, _codegen_units = ctx.attr._codegen_units[BuildSettingInfo].value, _experimental_use_allocator_libraries_with_mangled_symbols = ctx.attr.experimental_use_allocator_libraries_with_mangled_symbols, _experimental_use_allocator_libraries_with_mangled_symbols_setting = ctx.attr._experimental_use_allocator_libraries_with_mangled_symbols_setting[BuildSettingInfo].value, @@ -668,6 +675,9 @@ rust_toolchain = rule( doc = "The extension for binaries created from rustc.", mandatory = True, ), + "bootstrapping": attr.bool( + doc = "Internal attribute, set when bootstrapping process_wrapper. Do not use.", + ), "cargo": attr.label( doc = "The location of the `cargo` binary. Can be a direct source or a filegroup containing one item.", allow_single_file = True, @@ -801,6 +811,13 @@ rust_toolchain = rule( "per_crate_rustc_flags": attr.string_list( doc = "Extra flags to pass to rustc in non-exec configuration", ), + "process_wrapper": attr.label( + doc = "A process wrapper for running rustc on all platforms.", + executable = True, + allow_single_file = True, + cfg = "exec", + mandatory = True, + ), "require_explicit_unstable_features": attr.label( default = Label( "//rust/settings:require_explicit_unstable_features", @@ -951,6 +968,7 @@ rust_toolchain( rust_doc = "@rust_cpuX//:rustdoc", rust_std = "@rust_cpuX//:rust_std", rustc = "@rust_cpuX//:rustc", + process_wrapper = "@rules_rust//util/process_wrapper", rustc_lib = "@rust_cpuX//:rustc_lib", staticlib_ext = ".a", stdlib_linkflags = ["-lpthread", "-ldl"], @@ -980,6 +998,7 @@ To use a platform-specific linker, you can use a `select()` in the `linker` attr rust_toolchain( name = "rust_toolchain_impl", # ... other attributes ... + process_wrapper = "@rules_rust//util/process_wrapper", linker = select({ "@platforms//os:linux": "//tools:rust-lld-linux", "@platforms//os:windows": "//tools:rust-lld-windows", diff --git a/test/integration/cc_common_link/unit/cc_common_link_test.bzl b/test/integration/cc_common_link/unit/cc_common_link_test.bzl index 7a1c52403f..4525a31195 100644 --- a/test/integration/cc_common_link/unit/cc_common_link_test.bzl +++ b/test/integration/cc_common_link/unit/cc_common_link_test.bzl @@ -328,6 +328,7 @@ def _codegen_units_test_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust-lld", linker_type = "direct", staticlib_ext = ".a", diff --git a/test/toolchain/toolchain_test.bzl b/test/toolchain/toolchain_test.bzl index 3140064b32..8dda2f79aa 100644 --- a/test/toolchain/toolchain_test.bzl +++ b/test/toolchain/toolchain_test.bzl @@ -188,6 +188,7 @@ def _define_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust_lld", staticlib_ext = ".a", stdlib_linkflags = [], @@ -201,6 +202,7 @@ def _define_targets(): name = "extra_flags_toolchain", toolchain = ":rust_extra_flags_toolchain", toolchain_type = "@rules_rust//rust:toolchain", + target_settings = ["@rules_rust//rust/private:bootstrapped"], ) extra_toolchain_wrapper( diff --git a/test/unit/consistent_crate_name/with_modified_crate_name.bzl b/test/unit/consistent_crate_name/with_modified_crate_name.bzl index 7148b0f0b8..295a170478 100644 --- a/test/unit/consistent_crate_name/with_modified_crate_name.bzl +++ b/test/unit/consistent_crate_name/with_modified_crate_name.bzl @@ -64,12 +64,6 @@ with_modified_crate_name = rule( "_error_format": attr.label( default = Label("//rust/settings:error_format"), ), - "_process_wrapper": attr.label( - default = Label("//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), }, toolchains = [ "@rules_rust//rust:toolchain_type", diff --git a/test/unit/force_all_deps_direct/generator.bzl b/test/unit/force_all_deps_direct/generator.bzl index 09fba1f423..5aa0cdd445 100644 --- a/test/unit/force_all_deps_direct/generator.bzl +++ b/test/unit/force_all_deps_direct/generator.bzl @@ -81,12 +81,6 @@ generator = rule( "_error_format": attr.label( default = Label("//rust/settings:error_format"), ), - "_process_wrapper": attr.label( - default = Label("//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), }, toolchains = [ "@rules_rust//rust:toolchain_type", diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index 89b75b1850..12d91b096b 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -97,12 +97,6 @@ wrap = rule( "_error_format": attr.label( default = Label("//rust/settings:error_format"), ), - "_process_wrapper": attr.label( - default = Label("//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), }, toolchains = [ "@rules_rust//rust:toolchain", diff --git a/test/unit/toolchain/toolchain_test.bzl b/test/unit/toolchain/toolchain_test.bzl index 67232aadef..858a09f6a6 100644 --- a/test/unit/toolchain/toolchain_test.bzl +++ b/test/unit/toolchain/toolchain_test.bzl @@ -140,6 +140,7 @@ def _define_test_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust_lld", staticlib_ext = ".a", stdlib_linkflags = [], @@ -156,6 +157,7 @@ def _define_test_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust_lld", staticlib_ext = ".a", stdlib_linkflags = [], @@ -170,6 +172,7 @@ def _define_test_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust_lld", staticlib_ext = ".a", stdlib_linkflags = [], @@ -192,6 +195,7 @@ def _define_test_targets(): rust_doc = ":mock_rustdoc", rust_std = ":std_libs", rustc = ":mock_rustc", + process_wrapper = "@rules_rust//util/process_wrapper", linker = ":mock_rust_lld", staticlib_ext = ".a", stdlib_linkflags = ["test:$(location :stdlib_srcs)", "test:sysroot=$(RUST_SYSROOT)"], diff --git a/util/process_wrapper/BUILD.bazel b/util/process_wrapper/BUILD.bazel index dd8bd2992d..9c5826cdf9 100644 --- a/util/process_wrapper/BUILD.bazel +++ b/util/process_wrapper/BUILD.bazel @@ -20,6 +20,10 @@ rust_binary_without_process_wrapper( "@platforms//os:windows": ["-Cstrip=debuginfo"], "//conditions:default": [], }), + target_compatible_with = select({ + "@rules_rust//rust/private:bootstrapping": [], + "//conditions:default": ["@platforms//:incompatible"], + }), visibility = ["//visibility:public"], deps = [ "@rules_rust_tinyjson//:tinyjson", From e8cbbe58e40503bdceef2f89b643390ddcedee1b Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 26 Apr 2026 16:12:11 -0400 Subject: [PATCH 26/50] Capture process wrapper exit codes --- rust/private/clippy.bzl | 7 ++++++- util/process_wrapper/main.rs | 15 ++++++++++++++- util/process_wrapper/options.rs | 10 ++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index cdc7e25cd8..084a6fd7bd 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -101,7 +101,7 @@ def get_clippy_ready_crate_info(target, aspect_ctx = None): else: return None -def rust_clippy_action(ctx, clippy_executable, crate_info, config, output = None, success_marker = None, cap_at_warnings = False, extra_clippy_flags = [], error_format = None, clippy_diagnostics_file = None): +def rust_clippy_action(ctx, clippy_executable, crate_info, config, output = None, success_marker = None, cap_at_warnings = False, extra_clippy_flags = [], error_format = None, clippy_diagnostics_file = None, captured_exit_code_file = None): """Run clippy with the specified parameters. Args: @@ -115,6 +115,7 @@ def rust_clippy_action(ctx, clippy_executable, crate_info, config, output = None extra_clippy_flags (List[str]): A list of extra options to pass to clippy. If not set, every warnings will be turned into errors error_format (str): Which error format to use. Must be acceptable by rustc: https://doc.rust-lang.org/beta/rustc/command-line-arguments.html#--error-format-control-how-errors-are-produced clippy_diagnostics_file (File): File to output diagnostics to. If None, no diagnostics will be written + captured_exit_code_file (File): File to write clippy's exit code to. If set, the process wrapper exits successfully even when clippy fails. Returns: None @@ -199,6 +200,10 @@ def rust_clippy_action(ctx, clippy_executable, crate_info, config, output = None args.process_wrapper_flags.add("--touch-file", success_marker) outputs.append(success_marker) + if captured_exit_code_file != None: + args.process_wrapper_flags.add("--captured-exit-code-file", captured_exit_code_file) + outputs.append(captured_exit_code_file) + if clippy_flags or lint_files: args.rustc_flags.add_all(clippy_flags) else: diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 8b10955464..3139b45a85 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -373,6 +373,15 @@ fn main() -> Result<(), ProcessWrapperError> { .wait() .map_err(|e| ProcessWrapperError(format!("failed to wait for child process: {}", e)))?; let code = status.code().unwrap_or(1); + if let Some(exit_code_file) = &opts.captured_exit_code_file { + fs::write(exit_code_file, format!("{code}\n")).map_err(|e| { + ProcessWrapperError(format!( + "failed to write captured exit code to {}: {}", + exit_code_file, e + )) + })?; + } + if code == 0 { if let Some(tf) = opts.touch_file { OpenOptions::new() @@ -396,7 +405,11 @@ fn main() -> Result<(), ProcessWrapperError> { let _ = fs::remove_dir_all(path); } - exit(code) + if opts.captured_exit_code_file.is_some() { + Ok(()) + } else { + exit(code) + } } #[cfg(test)] diff --git a/util/process_wrapper/options.rs b/util/process_wrapper/options.rs index 7ec7b535ab..3cee829179 100644 --- a/util/process_wrapper/options.rs +++ b/util/process_wrapper/options.rs @@ -41,6 +41,9 @@ pub(crate) struct Options { pub(crate) stdout_file: Option, // If set, redirects the child process stderr to this file. pub(crate) stderr_file: Option, + // If set, writes the child process exit code to this file and exits + // successfully after the child process completes. + pub(crate) captured_exit_code_file: Option, // If set, also logs all unprocessed output from the rustc output to this file. // Meant to be used to get json output out of rustc for tooling usage. pub(crate) output_file: Option, @@ -61,6 +64,7 @@ pub(crate) fn options() -> Result { let mut copy_output_raw = None; let mut stdout_file = None; let mut stderr_file = None; + let mut captured_exit_code_file = None; let mut output_file = None; let mut rustc_output_format_raw = None; let mut flags = Flags::new(); @@ -104,6 +108,11 @@ pub(crate) fn options() -> Result { "Redirect subprocess stderr in this file.", &mut stderr_file, ); + flags.define_flag( + "--captured-exit-code-file", + "Write subprocess exit code to this file and exit successfully.", + &mut captured_exit_code_file, + ); flags.define_flag( "--output-file", "Log all unprocessed subprocess stderr in this file.", @@ -285,6 +294,7 @@ pub(crate) fn options() -> Result { copy_output, stdout_file, stderr_file, + captured_exit_code_file, output_file, rustc_output_format, }) From a8ef2b7613029de201990a658777009807cd6d30 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 27 Apr 2026 15:19:01 -0400 Subject: [PATCH 27/50] Provision hermetic SDKROOT --- rust/defs.bzl | 14 ++++++++++++-- rust/private/rust.bzl | 4 ++++ rust/private/rustc.bzl | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/rust/defs.bzl b/rust/defs.bzl index 8c6c14813d..d579c5e324 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -79,6 +79,16 @@ load( _rust_unpretty_aspect = "rust_unpretty_aspect", ) +_MACOS_SDKROOT_BY_PLATFORM = select({ + "@platforms//os:osx": "@macos_sdk//sysroot", + "//conditions:default": None, +}) + +def _add_macos_sdkroot(kwargs): + if kwargs.get("macos_sdkroot") == None: + kwargs["macos_sdkroot"] = _MACOS_SDKROOT_BY_PLATFORM + return kwargs + def _rule_wrapper(rule): def _wrapped(name, deps = [], proc_macro_deps = [], **kwargs): rule( @@ -86,7 +96,7 @@ def _rule_wrapper(rule): deps = deps + proc_macro_deps, # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. proc_macro_deps = deps + proc_macro_deps, - **kwargs + **_add_macos_sdkroot(kwargs) ) return _wrapped @@ -99,7 +109,7 @@ def _symbolic_rule_wrapper(rule, macro_fn): deps = deps + proc_macro_deps, # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. proc_macro_deps = deps + proc_macro_deps, - **kwargs + **_add_macos_sdkroot(kwargs) ) return macro_fn( diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 7a5817b774..8b685a025a 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -751,6 +751,10 @@ _COMMON_ATTRS = { """), allow_files = True, ), + "macos_sdkroot": attr.label( + doc = "Optional macOS SDK root to expose to rustc as SDKROOT when linking macOS targets.", + allow_files = True, + ), "deps": attr.label_list( doc = dedent("""\ List of other libraries to be linked to this library target. diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index b87ef5d4c0..5edfc27e7a 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -541,6 +541,16 @@ def get_linker_and_args(ctx, crate_type, toolchain, cc_toolchain, feature_config return ld, ld_is_direct_driver, link_args, link_env +def _apple_sdkroot(ctx, attr): + macos_sdkroot = getattr(attr, "macos_sdkroot", None) + if macos_sdkroot == None: + return None + + return ctx.expand_location( + "${{pwd}}/$(execpath {})".format(macos_sdkroot.label), + targets = [macos_sdkroot], + ) + def symlink_for_ambiguous_lib(actions, toolchain, crate_info, lib): """Constructs a disambiguating symlink for a library dependency. @@ -1287,6 +1297,10 @@ def construct_arguments( ) env.update(link_env) + macos_sdkroot = _apple_sdkroot(ctx, attr) + if macos_sdkroot != None: + env["SDKROOT"] = macos_sdkroot + rustc_flags.add(ld, format = "--codegen=linker=%s") # Split link args into individual "--codegen=link-arg=" flags to handle nested spaces. From 42d7dcb1a05b02898ce444617d3695dd33e2dfec Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 29 Apr 2026 21:27:30 -0400 Subject: [PATCH 28/50] Internalize macos SDK dep --- rust/defs.bzl | 14 ++------------ rust/private/BUILD.bazel | 14 ++++++++++++++ rust/private/rust.bzl | 1 + rust/private/rustc.bzl | 16 +++++++++++----- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/rust/defs.bzl b/rust/defs.bzl index d579c5e324..8c6c14813d 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -79,16 +79,6 @@ load( _rust_unpretty_aspect = "rust_unpretty_aspect", ) -_MACOS_SDKROOT_BY_PLATFORM = select({ - "@platforms//os:osx": "@macos_sdk//sysroot", - "//conditions:default": None, -}) - -def _add_macos_sdkroot(kwargs): - if kwargs.get("macos_sdkroot") == None: - kwargs["macos_sdkroot"] = _MACOS_SDKROOT_BY_PLATFORM - return kwargs - def _rule_wrapper(rule): def _wrapped(name, deps = [], proc_macro_deps = [], **kwargs): rule( @@ -96,7 +86,7 @@ def _rule_wrapper(rule): deps = deps + proc_macro_deps, # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. proc_macro_deps = deps + proc_macro_deps, - **_add_macos_sdkroot(kwargs) + **kwargs ) return _wrapped @@ -109,7 +99,7 @@ def _symbolic_rule_wrapper(rule, macro_fn): deps = deps + proc_macro_deps, # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. proc_macro_deps = deps + proc_macro_deps, - **_add_macos_sdkroot(kwargs) + **kwargs ) return macro_fn( diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index 7d8fa2cbcd..ab65a94e16 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -12,6 +12,20 @@ exports_files([ "test_sharding_wrapper.sh", ]) +filegroup( + name = "empty_macos_sdkroot", + visibility = ["//visibility:public"], +) + +alias( + name = "default_macos_sdkroot", + actual = select({ + "@platforms//os:osx": "@macos_sdk//sysroot", + "//conditions:default": ":empty_macos_sdkroot", + }), + visibility = ["//visibility:public"], +) + bzl_library( name = "bazel_tools_bzl_lib", srcs = ["@bazel_tools//tools:bzl_srcs"], diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 8b685a025a..263d674577 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -754,6 +754,7 @@ _COMMON_ATTRS = { "macos_sdkroot": attr.label( doc = "Optional macOS SDK root to expose to rustc as SDKROOT when linking macOS targets.", allow_files = True, + default = Label("//rust/private:default_macos_sdkroot"), ), "deps": attr.label_list( doc = dedent("""\ diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 5edfc27e7a..3c3417c2c0 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -541,15 +541,18 @@ def get_linker_and_args(ctx, crate_type, toolchain, cc_toolchain, feature_config return ld, ld_is_direct_driver, link_args, link_env +_EMPTY_MACOS_SDKROOT = Label("//rust/private:empty_macos_sdkroot") + def _apple_sdkroot(ctx, attr): macos_sdkroot = getattr(attr, "macos_sdkroot", None) - if macos_sdkroot == None: + if macos_sdkroot == None or macos_sdkroot.label == _EMPTY_MACOS_SDKROOT: return None - return ctx.expand_location( - "${{pwd}}/$(execpath {})".format(macos_sdkroot.label), - targets = [macos_sdkroot], - ) + files = macos_sdkroot[DefaultInfo].files.to_list() + if len(files) != 1: + fail("macos_sdkroot must provide exactly one file") + + return "${{pwd}}/{}".format(files[0].path) def symlink_for_ambiguous_lib(actions, toolchain, crate_info, lib): """Constructs a disambiguating symlink for a library dependency. @@ -807,6 +810,9 @@ def collect_inputs( if linker_script: nolinkstamp_compile_direct_inputs.append(linker_script) + if hasattr(files, "macos_sdkroot"): + nolinkstamp_compile_direct_inputs += files.macos_sdkroot + if not cc_toolchain: runtime_libs = depset() elif crate_info.type in ["dylib", "cdylib"]: From 242b2f8708e7ecbbb09f62de941490af7021ed86 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 2 May 2026 08:15:10 -0400 Subject: [PATCH 29/50] rustdoc_test should not do compilation at test time --- rust/private/rustc.bzl | 57 +++- rust/private/rustdoc.bzl | 6 +- rust/private/rustdoc/BUILD.bazel | 12 +- rust/private/rustdoc/rustdoc_test_runner.cc | 49 ++++ rust/private/rustdoc/rustdoc_test_writer.rs | 291 -------------------- rust/private/rustdoc_test.bzl | 140 +++------- test/unit/rustdoc/rustdoc_unit_test.bzl | 62 +++++ 7 files changed, 193 insertions(+), 424 deletions(-) create mode 100644 rust/private/rustdoc/rustdoc_test_runner.cc delete mode 100644 rust/private/rustdoc/rustdoc_test_writer.rs diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 3c3417c2c0..7c7ad1c8fd 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -718,7 +718,8 @@ def collect_inputs( stamp = False, force_depend_on_objects = False, experimental_use_cc_common_link = False, - include_link_flags = True): + include_link_flags = True, + force_link_inputs = False): """Gather's the inputs and required input information for a rustc action Args: @@ -741,6 +742,8 @@ def collect_inputs( experimental_use_cc_common_link (bool, optional): Whether rules_rust uses cc_common.link to link rust binaries. include_link_flags (bool, optional): Whether to include flags like `-l` that instruct the linker to search for a library. + force_link_inputs (bool, optional): Whether to collect linker inputs even when the crate type would + normally be handled as a non-linking compile action. Returns: tuple: A tuple: A tuple of the following items: @@ -774,7 +777,7 @@ def collect_inputs( # flattened on each transitive rust_library dependency. libs_from_linker_inputs = [] ambiguous_libs = {} - if crate_info.type not in ("lib", "rlib"): + if crate_info.type not in ("lib", "rlib") or force_link_inputs: linker_inputs = dep_info.transitive_noncrates.to_list() ambiguous_libs = _disambiguate_libs(ctx.actions, toolchain, crate_info, dep_info, use_pic) libs_from_linker_inputs = _collect_libs_from_linker_inputs(linker_inputs, use_pic) + [ @@ -1322,18 +1325,27 @@ def construct_arguments( else: rustc_flags.add("--codegen=link-arg=-Wl,-oso_prefix,${pwd}/") + # rustdoc tests compile doctest binaries through rustdoc instead of + # rustc. That frontend accepts native libraries as linker args, not + # rustc's `-lstatic`/`-ldylib` forms. + link_libraries_as_link_args = add_flags_for_binary and include_link_flags and not emit + native_link_crate_type = crate_info.type + if link_libraries_as_link_args and native_link_crate_type in ["lib", "rlib"]: + native_link_crate_type = "bin" + _add_native_link_flags( rustc_flags, dep_info, linkstamp_outs, ambiguous_libs, - crate_info.type, + native_link_crate_type, toolchain, cc_toolchain, feature_configuration, compilation_mode, ld_is_direct_driver, include_link_flags = include_link_flags, + link_libraries_as_link_args = link_libraries_as_link_args, ) use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) @@ -2520,8 +2532,20 @@ def portable_link_flags( def _add_user_link_flags(ret, linker_input): ret.extend(["--codegen=link-arg={}".format(flag) for flag in linker_input.user_link_flags]) +def _link_arg_compatible_link_flags(flags, link_libraries_as_link_args): + if not link_libraries_as_link_args: + return flags + + ret = [] + for flag in flags: + if flag.startswith("-Clink-arg=") or flag.startswith("--codegen=link-arg="): + ret.append(flag) + elif flag.startswith("-ldylib="): + ret.append("-Clink-arg=-l" + flag[len("-ldylib="):]) + return ret + def _make_link_flags_windows(make_link_flags_args, flavor_msvc, use_direct_driver): - linker_input, use_pic, ambiguous_libs, include_link_flags = make_link_flags_args + linker_input, use_pic, ambiguous_libs, include_link_flags, link_libraries_as_link_args = make_link_flags_args ret = [] prefix = "" if use_direct_driver else "-Wl," for lib in linker_input.libraries: @@ -2544,7 +2568,7 @@ def _make_link_flags_windows(make_link_flags_args, flavor_msvc, use_direct_drive if flag in ("-pthread", "-lpthread"): continue ret.append("--codegen=link-arg={}".format(flag)) - return ret + return _link_arg_compatible_link_flags(ret, link_libraries_as_link_args) def _make_link_flags_windows_msvc(make_link_flags_args, use_direct_driver): return _make_link_flags_windows(make_link_flags_args, flavor_msvc = True, use_direct_driver = use_direct_driver) @@ -2553,7 +2577,7 @@ def _make_link_flags_windows_gnu(make_link_flags_args, use_direct_driver): return _make_link_flags_windows(make_link_flags_args, flavor_msvc = False, use_direct_driver = use_direct_driver) def _make_link_flags_darwin(make_link_flags_args, use_direct_driver): - linker_input, use_pic, ambiguous_libs, include_link_flags = make_link_flags_args + linker_input, use_pic, ambiguous_libs, include_link_flags, link_libraries_as_link_args = make_link_flags_args ret = [] prefix = "" if use_direct_driver else "-Wl," for lib in linker_input.libraries: @@ -2565,10 +2589,10 @@ def _make_link_flags_darwin(make_link_flags_args, use_direct_driver): elif include_link_flags: ret.extend(portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name_default, for_darwin = True)) _add_user_link_flags(ret, linker_input) - return ret + return _link_arg_compatible_link_flags(ret, link_libraries_as_link_args) def _make_link_flags_default(make_link_flags_args, use_direct_driver): - linker_input, use_pic, ambiguous_libs, include_link_flags = make_link_flags_args + linker_input, use_pic, ambiguous_libs, include_link_flags, link_libraries_as_link_args = make_link_flags_args ret = [] prefix = "" if use_direct_driver else "-Wl," for lib in linker_input.libraries: @@ -2581,7 +2605,7 @@ def _make_link_flags_default(make_link_flags_args, use_direct_driver): elif include_link_flags: ret.extend(portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name_default)) _add_user_link_flags(ret, linker_input) - return ret + return _link_arg_compatible_link_flags(ret, link_libraries_as_link_args) def _make_link_flags_default_indirect(make_link_flags_args): return _make_link_flags_default(make_link_flags_args, False) @@ -2638,7 +2662,7 @@ def _get_make_link_flag_funcs(target_os, target_abi, use_direct_link_driver): return (make_link_flags, get_lib_name) def _libraries_dirnames(make_link_flags_args): - link_input, use_pic, _, _ = make_link_flags_args + link_input, use_pic, _, _, _ = make_link_flags_args # De-duplicate names. return depset([get_preferred_artifact(lib, use_pic).dirname for lib in link_input.libraries]).to_list() @@ -2654,7 +2678,8 @@ def _add_native_link_flags( feature_configuration, compilation_mode, use_direct_link_driver, - include_link_flags = True): + include_link_flags = True, + link_libraries_as_link_args = False): """Adds linker flags for all dependencies of the current target. Args: @@ -2669,6 +2694,8 @@ def _add_native_link_flags( compilation_mode (bool): The compilation mode for this build. use_direct_link_driver (bool): Whether the linker is a direct driver (e.g. `ld`, `wasm-ld`) vs a wrapper (e.g. `clang`, `gcc`). include_link_flags (bool, optional): Whether to include flags like `-l` that instruct the linker to search for a library. + link_libraries_as_link_args (bool, optional): Emit library search flags through linker args instead of + rustc's `-lstatic`/`-ldylib` forms. """ if crate_type in ["lib", "rlib"]: return @@ -2680,9 +2707,11 @@ def _add_native_link_flags( target_abi = toolchain.target_abi, use_direct_link_driver = use_direct_link_driver, ) + dynamic_runtime_link_format = "-Clink-arg=-l%s" if link_libraries_as_link_args else "-ldylib=%s" + static_runtime_link_format = "-Clink-arg=-l%s" if link_libraries_as_link_args else "-lstatic=%s" # TODO(hlopko): Remove depset flattening by using lambdas once we are on >=Bazel 5.0 - make_link_flags_args = [(arg, use_pic, ambiguous_libs, include_link_flags) for arg in dep_info.transitive_noncrates.to_list()] + make_link_flags_args = [(arg, use_pic, ambiguous_libs, include_link_flags, link_libraries_as_link_args) for arg in dep_info.transitive_noncrates.to_list()] args.add_all(make_link_flags_args, map_each = _libraries_dirnames, uniquify = True, format_each = "-Lnative=%s") if ambiguous_libs: # If there are ambiguous libs, the disambiguation symlinks to them are @@ -2715,7 +2744,7 @@ def _add_native_link_flags( args.add_all( cc_toolchain.dynamic_runtime_lib(feature_configuration = feature_configuration), map_each = get_lib_name, - format_each = "-ldylib=%s", + format_each = dynamic_runtime_link_format, ) else: # For all other crate types we want to link C++ runtime library statically @@ -2729,7 +2758,7 @@ def _add_native_link_flags( args.add_all( cc_toolchain.static_runtime_lib(feature_configuration = feature_configuration), map_each = get_lib_name, - format_each = "-lstatic=%s", + format_each = static_runtime_link_format, ) def _get_dirname(file): diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 074b7fec46..165a0d9aa8 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -118,6 +118,7 @@ def rustdoc_compile_action( # If this is a rustdoc test, we need to depend on rlibs rather than .rmeta. force_depend_on_objects = is_test, include_link_flags = False, + force_link_inputs = is_test, ) # Since this crate is not actually producing the output described by the @@ -131,7 +132,7 @@ def rustdoc_compile_action( attr = ctx.attr, file = ctx.file, toolchain = toolchain, - tool_path = toolchain.rust_doc.short_path if is_test else toolchain.rust_doc.path, + tool_path = toolchain.rust_doc.path, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, crate_info = rustdoc_crate_info, @@ -146,7 +147,7 @@ def rustdoc_compile_action( emit = [], remap_path_prefix = None, add_flags_for_binary = True, - include_link_flags = False, + include_link_flags = is_test, force_depend_on_objects = is_test, skip_expanding_rustc_env = True, ) @@ -168,6 +169,7 @@ def rustdoc_compile_action( inputs = all_inputs, env = env, arguments = args.all, + process_wrapper_flags = args.process_wrapper_flags, tools = [toolchain.rust_doc], ) diff --git a/rust/private/rustdoc/BUILD.bazel b/rust/private/rustdoc/BUILD.bazel index 85cda1cd7e..ddfabcd03d 100644 --- a/rust/private/rustdoc/BUILD.bazel +++ b/rust/private/rustdoc/BUILD.bazel @@ -1,12 +1,8 @@ -load("//rust:defs.bzl", "rust_binary") +load("@rules_cc//cc:defs.bzl", "cc_binary") package(default_visibility = ["//visibility:public"]) -rust_binary( - name = "rustdoc_test_writer", - srcs = ["rustdoc_test_writer.rs"], - edition = "2018", - deps = [ - "//rust/runfiles", - ], +cc_binary( + name = "rustdoc_test_runner", + srcs = ["rustdoc_test_runner.cc"], ) diff --git a/rust/private/rustdoc/rustdoc_test_runner.cc b/rust/private/rustdoc/rustdoc_test_runner.cc new file mode 100644 index 0000000000..0ffd7ce725 --- /dev/null +++ b/rust/private/rustdoc/rustdoc_test_runner.cc @@ -0,0 +1,49 @@ +#include +#include +#include + +namespace { + +bool CopyFileToStream(const std::string &path, std::ostream &out) { + std::ifstream file(path, std::ios::binary); + if (!file) { + std::cerr << "failed to open rustdoc test output file: " << path << "\n"; + return false; + } + out << file.rdbuf(); + return true; +} + +int ReadExitCode(const std::string &path) { + std::ifstream file(path); + if (!file) { + std::cerr << "failed to open rustdoc test exit code file: " << path << "\n"; + return -1; + } + + int exit_code = -1; + if (!(file >> exit_code) || exit_code < 0 || exit_code > 255) { + std::cerr << "invalid rustdoc test exit code file: " << path << "\n"; + return -1; + } + + return exit_code; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc < 1 || argv[0] == nullptr || argv[0][0] == '\0') { + std::cerr << "failed to determine rustdoc test executable path\n"; + return 1; + } + + const std::string test_executable(argv[0]); + + bool ok = true; + ok = CopyFileToStream(test_executable + ".rustdoc_test.stdout", std::cout) && ok; + ok = CopyFileToStream(test_executable + ".rustdoc_test.stderr", std::cerr) && ok; + + const int exit_code = ReadExitCode(test_executable + ".rustdoc_test.exit_code"); + return ok && exit_code >= 0 ? exit_code : 1; +} diff --git a/rust/private/rustdoc/rustdoc_test_writer.rs b/rust/private/rustdoc/rustdoc_test_writer.rs deleted file mode 100644 index aa3d97994a..0000000000 --- a/rust/private/rustdoc/rustdoc_test_writer.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! A utility for writing scripts for use as test executables intended to match the -//! subcommands of Bazel build actions so `rustdoc --test`, which builds and tests -//! code in a single call, can be run as a test target in a hermetic manner. - -use std::cmp::Reverse; -use std::collections::{BTreeMap, BTreeSet}; -use std::env; -use std::fs; -use std::io::{BufRead, BufReader}; -use std::path::{Path, PathBuf}; - -#[derive(Debug)] -struct Options { - /// A list of environment variable keys to parse from the build action env. - env_keys: BTreeSet, - - /// A list of substrings to strip from [Options::action_argv]. - strip_substrings: Vec, - - /// The path where the script should be written. - output: PathBuf, - - /// If Bazel generated a params file, we may need to strip roots from it. - /// This is the path where we will output our stripped params file. - optional_output_params_file: PathBuf, - - /// The `argv` of the configured rustdoc build action. - action_argv: Vec, -} - -/// Parse command line arguments -fn parse_args() -> Options { - let args: Vec = env::args().collect(); - let (writer_args, action_args) = { - let split = args - .iter() - .position(|arg| arg == "--") - .expect("Unable to find split identifier `--`"); - - // Converting each set into a vector makes them easier to parse in - // the absence of nightly features - let (writer, action) = args.split_at(split); - (writer.to_vec(), action.to_vec()) - }; - - // Remove the leading `--` which is expected to be the first - // item in `action_args` - debug_assert_eq!(action_args[0], "--"); - let action_argv = action_args[1..].to_vec(); - - let output = writer_args - .iter() - .find(|arg| arg.starts_with("--output=")) - .and_then(|arg| arg.splitn(2, '=').last()) - .map(PathBuf::from) - .expect("Missing `--output` argument"); - - let optional_output_params_file = writer_args - .iter() - .find(|arg| arg.starts_with("--optional_test_params=")) - .and_then(|arg| arg.splitn(2, '=').last()) - .map(PathBuf::from) - .expect("Missing `--optional_test_params` argument"); - - let (strip_substring_args, writer_args): (Vec, Vec) = writer_args - .into_iter() - .partition(|arg| arg.starts_with("--strip_substring=")); - - let mut strip_substrings: Vec = strip_substring_args - .into_iter() - .map(|arg| { - arg.splitn(2, '=') - .last() - .expect("--strip_substring arguments must have assignments using `=`") - .to_owned() - }) - .collect(); - - // Strip substrings should always be in reverse order of the length of each - // string so when filtering we know that the longer strings are checked - // first in order to avoid cases where shorter strings might match longer ones. - strip_substrings.sort_by_key(|b| Reverse(b.len())); - strip_substrings.dedup(); - - let env_keys = writer_args - .into_iter() - .filter(|arg| arg.starts_with("--action_env=")) - .map(|arg| { - arg.splitn(2, '=') - .last() - .expect("--env arguments must have assignments using `=`") - .to_owned() - }) - .collect(); - - Options { - env_keys, - strip_substrings, - output, - optional_output_params_file, - action_argv, - } -} - -/// Expand the Bazel Arg file and write it into our manually defined params file -fn expand_params_file(mut options: Options) -> Options { - let params_extension = if cfg!(target_family = "windows") { - ".rustdoc_test.bat-0.params" - } else { - ".rustdoc_test.sh-0.params" - }; - - // We always need to produce the params file, we might overwrite this later though - fs::write(&options.optional_output_params_file, b"unused") - .expect("Failed to write params file"); - - // extract the path for the params file, if it exists - let params_path = match options.action_argv.pop() { - // Found the params file! - Some(arg) if arg.starts_with('@') && arg.ends_with(params_extension) => { - let path_str = arg - .strip_prefix('@') - .expect("Checked that there is an @ prefix"); - PathBuf::from(path_str) - } - // No params file present, exit early - Some(arg) => { - options.action_argv.push(arg); - return options; - } - None => return options, - }; - - // read the params file - let params_file = fs::File::open(params_path).expect("Failed to read the rustdoc params file"); - let content: Vec<_> = BufReader::new(params_file) - .lines() - .map(|line| line.expect("failed to parse param as String")) - // Remove any substrings found in the argument - .map(|arg| { - let mut stripped_arg = arg; - options - .strip_substrings - .iter() - .for_each(|substring| stripped_arg = stripped_arg.replace(substring, "")); - stripped_arg - }) - .collect(); - - // add all arguments - fs::write(&options.optional_output_params_file, content.join("\n")) - .expect("Failed to write test runner"); - - // append the path of our new params file - let formatted_params_path = format!( - "@{}", - options - .optional_output_params_file - .to_str() - .expect("invalid UTF-8") - ); - options.action_argv.push(formatted_params_path); - - options -} - -/// Write a unix compatible test runner -fn write_test_runner_unix( - path: &Path, - env: &BTreeMap, - argv: &[String], - strip_substrings: &[String], -) { - let mut content = vec![ - "#!/usr/bin/env bash".to_owned(), - "".to_owned(), - // TODO: Instead of creating a symlink to mimic the behavior of - // --legacy_external_runfiles, this rule should be able to correctly - // sanitize the action args to run in a runfiles without this link. - "if [[ ! -e 'external' ]]; then ln -s ../ external ; fi".to_owned(), - "".to_owned(), - "exec env - \\".to_owned(), - ]; - - content.extend(env.iter().map(|(key, val)| format!("{key}='{val}' \\"))); - - let argv_str = argv - .iter() - // Remove any substrings found in the argument - .map(|arg| { - let mut stripped_arg = arg.to_owned(); - strip_substrings - .iter() - .for_each(|substring| stripped_arg = stripped_arg.replace(substring, "")); - stripped_arg - }) - .map(|arg| format!("'{arg}'")) - .collect::>() - .join(" "); - - content.extend(vec![argv_str, "".to_owned()]); - - fs::write(path, content.join("\n")).expect("Failed to write test runner"); -} - -/// Write a windows compatible test runner -fn write_test_runner_windows( - path: &Path, - env: &BTreeMap, - argv: &[String], - strip_substrings: &[String], -) { - let env_str = env - .iter() - .map(|(key, val)| format!("$env:{key}='{val}'")) - .collect::>() - .join(" ; "); - - let argv_str = argv - .iter() - // Remove any substrings found in the argument - .map(|arg| { - let mut stripped_arg = arg.to_owned(); - strip_substrings - .iter() - .for_each(|substring| stripped_arg = stripped_arg.replace(substring, "")); - stripped_arg - }) - .map(|arg| format!("'{arg}'")) - .collect::>() - .join(" "); - - let content = [ - "@ECHO OFF".to_owned(), - "".to_owned(), - // TODO: Instead of creating a symlink to mimic the behavior of - // --legacy_external_runfiles, this rule should be able to correctly - // sanitize the action args to run in a runfiles without this link. - "powershell.exe -c \"if (!(Test-Path .\\external)) { New-Item -Path .\\external -ItemType SymbolicLink -Value ..\\ }\"" - .to_owned(), - "".to_owned(), - format!("powershell.exe -c \"{env_str} ; & {argv_str}\""), - "".to_owned(), - ]; - - fs::write(path, content.join("\n")).expect("Failed to write test runner"); -} - -#[cfg(target_family = "unix")] -fn set_executable(path: &Path) { - use std::os::unix::prelude::PermissionsExt; - - let mut perm = fs::metadata(path) - .expect("Failed to get test runner metadata") - .permissions(); - - perm.set_mode(0o755); - fs::set_permissions(path, perm).expect("Failed to set permissions on test runner"); -} - -#[cfg(target_family = "windows")] -fn set_executable(_path: &Path) { - // Windows determines whether or not a file is executable via the PATHEXT - // environment variable. This function is a no-op for this platform. -} - -fn write_test_runner( - path: &Path, - env: &BTreeMap, - argv: &[String], - strip_substrings: &[String], -) { - if cfg!(target_family = "unix") { - write_test_runner_unix(path, env, argv, strip_substrings); - } else if cfg!(target_family = "windows") { - write_test_runner_windows(path, env, argv, strip_substrings); - } - - set_executable(path); -} - -fn main() { - let opt = parse_args(); - let opt = expand_params_file(opt); - - let env: BTreeMap = env::vars() - .filter(|(key, _)| opt.env_keys.iter().any(|k| k == key)) - .collect(); - - write_test_runner(&opt.output, &env, &opt.action_argv, &opt.strip_substrings); -} diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index d521ccd279..c8500d9735 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -20,83 +20,6 @@ load("//rust/private:providers.bzl", "CrateInfo") load("//rust/private:rustdoc.bzl", "rustdoc_compile_action") load("//rust/private:utils.bzl", "dedent", "filter_deps", "find_toolchain", "transform_deps") -def _construct_writer_arguments(ctx, test_runner, opt_test_params, action, crate_info): - """Construct arguments and environment variables specific to `rustdoc_test_writer`. - - This is largely solving for the fact that tests run from a runfiles directory - where actions run in an execroot. But it also tracks what environment variables - were explicitly added to the action. - - Args: - ctx (ctx): The rule's context object. - test_runner (File): The test_runner output file declared by `rustdoc_test`. - opt_test_params (File): An output file we can optionally use to store params for `rustdoc`. - action (struct): Action arguments generated by `rustdoc_compile_action`. - crate_info (CrateInfo): The provider of the crate who's docs are being tested. - - Returns: - tuple: A tuple of `rustdoc_test_writer` specific inputs - - Args: Arguments for the test writer - - dict: Required environment variables - """ - - writer_args = ctx.actions.args() - - # Track the output path where the test writer should write the test - writer_args.add("--output={}".format(test_runner.path)) - - # Track where the test writer should move "spilled" Args to - writer_args.add("--optional_test_params={}".format(opt_test_params.path)) - - # Track what environment variables should be written to the test runner - writer_args.add("--action_env=DEVELOPER_DIR") - writer_args.add("--action_env=PATHEXT") - writer_args.add("--action_env=SDKROOT") - writer_args.add("--action_env=SYSROOT") - for var in action.env.keys(): - writer_args.add("--action_env={}".format(var)) - - # Since the test runner will be running from a runfiles directory, the - # paths originally generated for the build action will not map to any - # files. To ensure rustdoc can find the appropriate dependencies, the - # file roots are identified and tracked for each dependency so it can be - # stripped from the test runner. - - # Collect and dedupe all of the file roots in a list before appending - # them to args to prevent generating a large amount of identical args - roots = [] - root = crate_info.output.root.path - if not root in roots: - roots.append(root) - for dep in crate_info.deps.to_list() + crate_info.proc_macro_deps.to_list(): - dep_crate_info = getattr(dep, "crate_info", None) - dep_dep_info = getattr(dep, "dep_info", None) - if dep_crate_info: - root = dep_crate_info.output.root.path - if not root in roots: - roots.append(root) - if dep_dep_info: - for direct_dep in dep_dep_info.direct_crates.to_list(): - root = direct_dep.dep.output.root.path - if not root in roots: - roots.append(root) - for transitive_dep in dep_dep_info.transitive_crates.to_list(): - root = transitive_dep.output.root.path - if not root in roots: - roots.append(root) - - for root in roots: - writer_args.add("--strip_substring={}/".format(root)) - - # Indicate that the rustdoc_test args are over. - writer_args.add("--") - - # Prepare for the process runner to ingest the rest of the arguments - # to match the expectations of `rustc_compile_action`. - writer_args.add(action.executable.short_path) - - return (writer_args, action.env) - def _rust_doc_test_impl(ctx): """The implementation for the `rust_doc_test` rule @@ -134,21 +57,19 @@ def _rust_doc_test_impl(ctx): owner = ctx.label, ) - if toolchain.target_os == "windows": - test_runner = ctx.actions.declare_file(ctx.label.name + ".rustdoc_test.bat") - else: - test_runner = ctx.actions.declare_file(ctx.label.name + ".rustdoc_test.sh") + test_runner_name = ctx.label.name + if ctx.executable._test_runner.extension: + test_runner_name += "." + ctx.executable._test_runner.extension - # Bazel will auto-magically spill params to a file, if they are too many for a given OSes shell - # (e.g. Windows ~32k, Linux ~2M). The executable script (aka test_runner) that gets generated, - # is run from the runfiles, which is separate from the params_file Bazel generates. To handle - # this case, we declare our own params file, that the test_writer will populate, if necessary - opt_test_params = ctx.actions.declare_file(ctx.label.name + ".rustdoc_opt_params", sibling = test_runner) + test_runner = ctx.actions.declare_file(test_runner_name) + stdout_file = ctx.actions.declare_file(test_runner.basename + ".rustdoc_test.stdout", sibling = test_runner) + stderr_file = ctx.actions.declare_file(test_runner.basename + ".rustdoc_test.stderr", sibling = test_runner) + exit_code_file = ctx.actions.declare_file(test_runner.basename + ".rustdoc_test.exit_code", sibling = test_runner) # Add the current crate as an extern for the compile action rustdoc_flags = [ "--extern", - "{}={}".format(crate_info.name, crate_info.output.short_path), + "{}={}".format(crate_info.name, crate_info.output.path), "--test", ] @@ -162,33 +83,34 @@ def _rust_doc_test_impl(ctx): is_test = True, ) - tools = action.tools + [action.executable] - - writer_args, env = _construct_writer_arguments( - ctx = ctx, - test_runner = test_runner, - opt_test_params = opt_test_params, - action = action, - crate_info = crate_info, - ) - - # Allow writer environment variables to override those from the action. - action.env.update(env) + action.process_wrapper_flags.add("--stdout-file", stdout_file) + action.process_wrapper_flags.add("--stderr-file", stderr_file) + action.process_wrapper_flags.add("--captured-exit-code-file", exit_code_file) ctx.actions.run( - mnemonic = "RustdocTestWriter", - progress_message = "Generating Rustdoc test runner for {}".format(ctx.attr.crate.label), - executable = ctx.executable._test_writer, + mnemonic = "RustdocTest", + progress_message = "Running Rustdoc test for %{label}", + executable = action.executable, inputs = action.inputs, - tools = tools, - arguments = [writer_args] + action.arguments, + tools = action.tools, + arguments = action.arguments, env = action.env, - outputs = [test_runner, opt_test_params], + outputs = [stdout_file, stderr_file, exit_code_file], + toolchain = Label("//rust:toolchain_type"), + ) + + ctx.actions.symlink( + output = test_runner, + target_file = ctx.executable._test_runner, + is_executable = True, ) + runfiles = ctx.runfiles(files = [stdout_file, stderr_file, exit_code_file]) + runfiles = runfiles.merge(ctx.attr._test_runner[DefaultInfo].default_runfiles) + return [DefaultInfo( files = depset([test_runner]), - runfiles = ctx.runfiles(files = tools + [opt_test_params], transitive_files = action.inputs), + runfiles = runfiles, executable = test_runner, )] @@ -235,10 +157,10 @@ rust_doc_test = rule( file of arguments to rustc: `@$(location //package:target)`. """), ), - "_test_writer": attr.label( - doc = "A binary used for writing script for use as the test executable.", + "_test_runner": attr.label( + doc = "A binary used for replaying rustdoc test build action results.", cfg = "exec", - default = Label("//rust/private/rustdoc:rustdoc_test_writer"), + default = Label("//rust/private/rustdoc:rustdoc_test_runner"), executable = True, ), }, diff --git a/test/unit/rustdoc/rustdoc_unit_test.bzl b/test/unit/rustdoc/rustdoc_unit_test.bzl index aa1f9b2f02..ccb17a7f8b 100644 --- a/test/unit/rustdoc/rustdoc_unit_test.bzl +++ b/test/unit/rustdoc/rustdoc_unit_test.bzl @@ -8,6 +8,7 @@ load( "//test/unit:common.bzl", "assert_action_mnemonic", "assert_argv_contains", + "assert_argv_contains_prefix", "assert_argv_contains_prefix_not", ) @@ -25,6 +26,16 @@ def _get_rustdoc_action(env, tut): return action +def _get_action_by_mnemonic(env, tut, mnemonic): + actions = [action for action in tut.actions if action.mnemonic == mnemonic] + asserts.equals( + env, + 1, + len(actions), + "Expected exactly one {} action, got {}".format(mnemonic, [action.mnemonic for action in tut.actions]), + ) + return actions[0] + def _common_rustdoc_checks(env, tut): action = _get_rustdoc_action(env, tut) @@ -154,6 +165,43 @@ def _rustdoc_with_json_error_format_test_impl(ctx): return analysistest.end(env) +def _rustdoc_test_runs_in_build_action_test_impl(ctx): + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + + action = _get_action_by_mnemonic(env, tut, "RustdocTest") + + assert_argv_contains(env, action, "--test") + assert_argv_contains(env, action, "--stdout-file") + assert_argv_contains(env, action, "--stderr-file") + assert_argv_contains(env, action, "--captured-exit-code-file") + + files = tut[DefaultInfo].files.to_list() + asserts.equals(env, 1, len(files)) + asserts.true( + env, + files[0].basename == "lib_doctest", + "Expected rust_doc_test to expose the test runner symlink, got {}".format(files[0].basename), + ) + + runfile_basenames = [file.basename for file in tut[DefaultInfo].default_runfiles.files.to_list()] + asserts.true(env, "lib_doctest.rustdoc_test.stdout" in runfile_basenames) + asserts.true(env, "lib_doctest.rustdoc_test.stderr" in runfile_basenames) + asserts.true(env, "lib_doctest.rustdoc_test.exit_code" in runfile_basenames) + + return analysistest.end(env) + +def _rustdoc_test_with_cc_link_flags_test_impl(ctx): + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + + action = _get_action_by_mnemonic(env, tut, "RustdocTest") + + assert_argv_contains_prefix(env, action, "-Lnative=") + assert_argv_contains(env, action, "-Clink-arg=-lcc_lib") + + return analysistest.end(env) + rustdoc_for_lib_test = analysistest.make(_rustdoc_for_lib_test_impl) rustdoc_for_bin_test = analysistest.make(_rustdoc_for_bin_test_impl) rustdoc_for_bin_with_cc_lib_test = analysistest.make(_rustdoc_for_bin_with_cc_lib_test_impl) @@ -168,6 +216,8 @@ rustdoc_zip_output_test = analysistest.make(_rustdoc_zip_output_test_impl) rustdoc_with_json_error_format_test = analysistest.make(_rustdoc_with_json_error_format_test_impl, config_settings = { str(Label("//rust/settings:error_format")): "json", }) +rustdoc_test_runs_in_build_action_test = analysistest.make(_rustdoc_test_runs_in_build_action_test_impl) +rustdoc_test_with_cc_link_flags_test = analysistest.make(_rustdoc_test_with_cc_link_flags_test_impl) def _target_maker(rule_fn, name, rustdoc_deps = [], rustdoc_proc_macro_deps = [], **kwargs): rule_fn( @@ -434,6 +484,16 @@ def rustdoc_test_suite(name): target_under_test = ":lib_doc", ) + rustdoc_test_runs_in_build_action_test( + name = "rustdoc_test_runs_in_build_action_test", + target_under_test = ":lib_doctest", + ) + + rustdoc_test_with_cc_link_flags_test( + name = "rustdoc_test_with_cc_link_flags_test", + target_under_test = ":lib_with_cc_doctest", + ) + native.filegroup( name = "lib_doc_zip", srcs = [":lib_doc.zip"], @@ -457,6 +517,8 @@ def rustdoc_test_suite(name): ":rustdoc_for_lib_with_cc_lib_test", ":rustdoc_with_args_test", ":rustdoc_with_json_error_format_test", + ":rustdoc_test_runs_in_build_action_test", + ":rustdoc_test_with_cc_link_flags_test", ":rustdoc_zip_output_test", ], ) From 720725bd94c76a9109a754302a76f8707451495d Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 3 May 2026 13:40:44 -0400 Subject: [PATCH 30/50] Fix CcInfo collection through CrateGroupInfo --- rust/private/rustc.bzl | 18 +++++++++++++++--- test/unit/cc_info/cc_info_test.bzl | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 7c7ad1c8fd..ecfdb5400c 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -2081,6 +2081,18 @@ def _collect_nonstatic_linker_inputs(cc_info): )) return shared_linker_inputs +def _collect_dep_cc_infos(dep): + cc_infos = [] + crate_group = getattr(dep, "crate_group_info", None) + if crate_group: + for group_dep in crate_group.dep_variant_infos.to_list(): + if group_dep.cc_info: + cc_infos.append(group_dep.cc_info) + elif dep.cc_info: + cc_infos.append(dep.cc_info) + + return cc_infos + def _add_lto_flags(ctx, toolchain, args, crate): """Adds flags to an Args object to configure LTO for 'rustc'. @@ -2200,18 +2212,18 @@ def establish_cc_info(ctx, attr, crate_info, toolchain, cc_toolchain, feature_co # Flattening is okay since crate_info.deps only records direct deps. for dep in crate_info.deps.to_list(): - if dep.cc_info: + for dep_cc_info in _collect_dep_cc_infos(dep): # A Rust staticlib or shared library doesn't need to propagate linker inputs # of its dependencies, except for shared libraries. if crate_info.type in ["cdylib", "staticlib"]: - shared_linker_inputs = _collect_nonstatic_linker_inputs(dep.cc_info) + shared_linker_inputs = _collect_nonstatic_linker_inputs(dep_cc_info) if shared_linker_inputs: linking_context = cc_common.create_linking_context( linker_inputs = depset(shared_linker_inputs), ) cc_infos.append(CcInfo(linking_context = linking_context)) else: - cc_infos.append(dep.cc_info) + cc_infos.append(dep_cc_info) if crate_info.type in ("rlib", "lib"): libstd_and_allocator_cc_info = _get_std_and_alloc_info(ctx, toolchain, crate_info) diff --git a/test/unit/cc_info/cc_info_test.bzl b/test/unit/cc_info/cc_info_test.bzl index 9b1eabce89..c371a0927b 100644 --- a/test/unit/cc_info/cc_info_test.bzl +++ b/test/unit/cc_info/cc_info_test.bzl @@ -54,6 +54,17 @@ def _collect_user_link_flags(env, tut): linker_inputs = cc_info.linking_context.linker_inputs.to_list() return [f for i in linker_inputs for f in i.user_link_flags] +def _collect_static_library_basenames(env, tut): + asserts.true(env, CcInfo in tut, "rust_library should provide CcInfo") + cc_info = tut[CcInfo] + linker_inputs = cc_info.linking_context.linker_inputs.to_list() + return [ + library_to_link.static_library.basename + for linker_input in linker_inputs + for library_to_link in linker_input.libraries + if library_to_link.static_library != None + ] + def _rlib_provides_cc_info_test_impl(ctx): env = analysistest.begin(ctx) tut = analysistest.target_under_test(env) @@ -111,6 +122,12 @@ def _crate_group_info_provides_cc_info_test_impl(ctx): len(tut[rust_common.dep_info].transitive_noncrates.to_list()) == 1, "crate_group_info should provide 1 non-crate transitive dependency", ) + static_library_basenames = _collect_static_library_basenames(env, tut) + asserts.true( + env, + "libcc_lib.a" in static_library_basenames, + "rust_library CcInfo should include static libraries from CrateGroupInfo deps, got {}".format(static_library_basenames), + ) return analysistest.end(env) rlib_provides_cc_info_test = analysistest.make( From 032a92bc57e13443c9c5bcfdc3810e5c14bd07cc Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 3 May 2026 13:55:17 -0400 Subject: [PATCH 31/50] Expose macOS SDK from llvm module --- MODULE.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 7ba52073b9..8418c4bc2a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -17,6 +17,10 @@ bazel_dep(name = "rules_cc", version = "0.2.4") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "apple_support", version = "1.24.1") +bazel_dep(name = "llvm", version = "0.7.7") + +osx = use_extension("@llvm//extensions:osx.bzl", "osx") +use_repo(osx, "macos_sdk") internal_deps = use_extension("//rust/private:internal_extensions.bzl", "i") use_repo( From 70e433bddff670ebca66b6a8dac61674894cc4cc Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 3 May 2026 14:19:42 -0400 Subject: [PATCH 32/50] Loongarch64 and sparc support --- test/unit/platform_triple/platform_triple_test.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/platform_triple/platform_triple_test.bzl b/test/unit/platform_triple/platform_triple_test.bzl index 0248782f51..2c66d9eb34 100644 --- a/test/unit/platform_triple/platform_triple_test.bzl +++ b/test/unit/platform_triple/platform_triple_test.bzl @@ -123,6 +123,9 @@ def _construct_known_triples_test_impl(ctx): _assert_parts(env, triple("aarch64-apple-ios-macabi"), "aarch64", "apple", "ios", "macabi") _assert_parts(env, triple("aarch64-fuchsia"), "aarch64", "unknown", "fuchsia", None) _assert_parts(env, triple("aarch64-unknown-linux-musl"), "aarch64", "unknown", "linux", "musl") + _assert_parts(env, triple("loongarch64-unknown-linux-gnu"), "loongarch64", "unknown", "linux", "gnu") + _assert_parts(env, triple("loongarch64-unknown-none"), "loongarch64", "unknown", "none", None) + _assert_parts(env, triple("sparc64-unknown-linux-gnu"), "sparc64", "unknown", "linux", "gnu") _assert_parts(env, triple("thumbv7em-none-eabi"), "thumbv7em", None, "none", "eabi") _assert_parts(env, triple("thumbv7em-none-eabihf"), "thumbv7em", None, "none", "eabihf") _assert_parts(env, triple("thumbv8m.main-none-eabi"), "thumbv8m.main", None, "none", "eabi") From cb08919f56397de9a5a678925638290ef2ac71b6 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 1 Apr 2026 15:10:09 -0400 Subject: [PATCH 33/50] Avoid vendored crates for rust-analyzer binary --- tools/rust_analyzer/BUILD.bazel | 136 +++++--------------------------- tools/rust_analyzer/aquery.rs | 2 +- tools/rust_analyzer/lib.rs | 2 +- 3 files changed, 20 insertions(+), 120 deletions(-) diff --git a/tools/rust_analyzer/BUILD.bazel b/tools/rust_analyzer/BUILD.bazel index 965eb6a964..6d56ffca26 100644 --- a/tools/rust_analyzer/BUILD.bazel +++ b/tools/rust_analyzer/BUILD.bazel @@ -1,118 +1,18 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//rust:defs.bzl", "rust_binary", "rust_clippy", "rust_library", "rust_test") -load("//tools/private:tool_utils.bzl", "aspect_repository") - -exports_files( - [ - "aquery.rs", - "bin/discover_rust_project.rs", - "bin/gen_rust_project.rs", - "bin/validate.rs", - "lib.rs", - "rust_project.rs", - ], - visibility = ["//visibility:public"], -) - -rust_binary( - name = "discover_bazel_rust_project", - srcs = ["bin/discover_rust_project.rs"], - edition = "2018", - rustc_env = { - "ASPECT_REPOSITORY": aspect_repository(), - }, - visibility = ["//visibility:public"], - deps = [ - ":gen_rust_project_lib", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) - -rust_binary( - name = "gen_rust_project", - srcs = ["bin/gen_rust_project.rs"], - edition = "2018", - rustc_env = { - "ASPECT_REPOSITORY": aspect_repository(), - }, - visibility = ["//visibility:public"], - deps = [ - ":gen_rust_project_lib", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) - -rust_binary( - name = "validate", - srcs = ["bin/validate.rs"], - data = [ - "//rust/toolchain:current_rust_analyzer_toolchain", - ], - edition = "2021", - rustc_env = { - "RUST_ANALYZER_RLOCATIONPATH": "$(RUST_ANALYZER_RLOCATIONPATH)", - }, - toolchains = ["//rust/toolchain:current_rust_analyzer_toolchain"], - visibility = ["//visibility:public"], - deps = [ - "//rust/runfiles", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - ], -) - -rust_library( - name = "gen_rust_project_lib", - srcs = glob( - ["**/*.rs"], - exclude = ["bin"], - ), - data = [ - "//rust/private:rust_analyzer_detect_sysroot", - ], - edition = "2018", - deps = [ - "//rust/runfiles", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) - -rust_test( - name = "gen_rust_project_lib_test", - crate = ":gen_rust_project_lib", - deps = [ - "//tools/rust_analyzer/3rdparty/crates:itertools", - ], -) - -rust_clippy( - name = "gen_rust_project_clippy", - testonly = True, - visibility = ["//visibility:private"], - deps = [ - ":gen_rust_project", - ], -) - -bzl_library( - name = "bzl_lib", - srcs = [], - visibility = ["//visibility:public"], - deps = ["//tools/rust_analyzer/3rdparty:bzl_lib"], -) +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "aquery.rs", + "lib.rs", + "rust_project.rs", + "bin/discover_rust_project.rs", + "bin/gen_rust_project.rs", + "bin/validate.rs", +]) + +alias(name = "discover_bazel_rust_project", actual = "@rules_rs//tools/rust_analyzer:discover_bazel_rust_project") +alias(name = "gen_rust_project", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project") +alias(name = "validate", actual = "@rules_rs//tools/rust_analyzer:validate") +alias(name = "gen_rust_project_lib", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_lib") +alias(name = "gen_rust_project_lib_test", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_lib_test") +alias(name = "gen_rust_project_clippy", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_clippy") +alias(name = "bzl_lib", actual = "@rules_rs//tools/rust_analyzer:bzl_lib") diff --git a/tools/rust_analyzer/aquery.rs b/tools/rust_analyzer/aquery.rs index f306aa5b75..c1ebeddbd1 100644 --- a/tools/rust_analyzer/aquery.rs +++ b/tools/rust_analyzer/aquery.rs @@ -104,7 +104,7 @@ pub fn get_crate_specs( .arg("--include_aspects") .arg("--include_artifacts") .arg(format!( - "--aspects={rules_rust_name}//rust:defs.bzl%rust_analyzer_aspect" + "--aspects={rules_rust_name}//tools/rust_analyzer:defs.bzl%rust_analyzer_aspect" )) .arg("--output_groups=rust_analyzer_crate_spec") .arg(format!( diff --git a/tools/rust_analyzer/lib.rs b/tools/rust_analyzer/lib.rs index 4ea641e6a0..93ca7cf3da 100644 --- a/tools/rust_analyzer/lib.rs +++ b/tools/rust_analyzer/lib.rs @@ -109,7 +109,7 @@ fn generate_crate_info( .arg("--norun_validations") .arg("--remote_download_all") .arg(format!( - "--aspects={rules_rust}//rust:defs.bzl%rust_analyzer_aspect" + "--aspects={rules_rust}//tools/rust_analyzer:defs.bzl%rust_analyzer_aspect" )) .arg("--output_groups=rust_analyzer_crate_spec,rust_generated_srcs,rust_analyzer_proc_macro_dylib,rust_analyzer_src") .args(targets) From b9f81123ebb2a778491c82db134bfc8065137ea1 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 4 May 2026 17:24:53 -0400 Subject: [PATCH 34/50] Don't force passing proc_macro_deps via macro --- rust/private/rust.bzl | 6 +++++- rust/private/utils.bzl | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 263d674577..7451ca94b5 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -784,6 +784,10 @@ _COMMON_ATTRS = { """), cfg = "exec", ), + "skip_deps_verification": attr.bool( + doc = "Internal only, do not use.", + default = False, + ), "require_explicit_unstable_features": attr.int( doc = ( "Whether to require all unstable features to be explicitly opted in to using " + @@ -1454,7 +1458,7 @@ rust_binary_without_process_wrapper = rule( "_allowlist_function_transition": attr.label( default = Label("//tools/allowlists/function_transition_allowlist"), ), - "_skip_deps_verification": attr.bool(default = True), + "skip_deps_verification": attr.bool(default = True), }), executable = True, fragments = ["cpp"], diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index 6ee717882e..bc52f85d0a 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -519,7 +519,7 @@ def filter_deps(ctx): Returns: deps and proc_macro_deps """ - if len(ctx.attr.deps) != len(ctx.attr.proc_macro_deps) and not getattr(ctx.attr, "_skip_deps_verification", False): + if len(ctx.attr.deps) != len(ctx.attr.proc_macro_deps) and not getattr(ctx.attr, "skip_deps_verification", False): fail("All deps should be passed to both `deps` and `proc_macro_deps`; please use the macros in //rust:defs.bzl") deps = [] From 01ad0e998cc5e0f31962318241939936af7d7cee Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 5 May 2026 11:37:24 -0400 Subject: [PATCH 35/50] Remove cargo build script rundir support --- cargo/private/cargo_build_script.bzl | 15 ---------- .../private/cargo_build_script_runner/bin.rs | 30 ++----------------- cargo/private/cargo_build_script_wrapper.bzl | 11 +++---- .../run_from_exec_root/BUILD.bazel | 17 ----------- .../run_from_exec_root/build.rs | 6 ---- .../run_from_exec_root/data.txt | 1 - .../run_from_exec_root/test.rs | 4 --- crate_universe/extensions.bzl | 3 -- crate_universe/private/crate.bzl | 3 -- crate_universe/src/config.rs | 7 ----- crate_universe/src/context/crate_context.rs | 7 ----- crate_universe/src/lockfile.rs | 2 +- crate_universe/src/rendering.rs | 6 +--- crate_universe/src/utils/starlark.rs | 3 -- .../src/utils/starlark/select_scalar.rs | 4 --- 15 files changed, 9 insertions(+), 110 deletions(-) delete mode 100644 cargo/tests/cargo_build_script/run_from_exec_root/BUILD.bazel delete mode 100644 cargo/tests/cargo_build_script/run_from_exec_root/build.rs delete mode 100644 cargo/tests/cargo_build_script/run_from_exec_root/data.txt delete mode 100644 cargo/tests/cargo_build_script/run_from_exec_root/test.rs diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_script.bzl index 9e68298330..66fb123e71 100644 --- a/cargo/private/cargo_build_script.bzl +++ b/cargo/private/cargo_build_script.bzl @@ -573,8 +573,6 @@ def _cargo_build_script_impl(ctx): args.add(link_flags, format = "--link_flags=%s") args.add(link_search_paths, format = "--link_search_paths=%s") args.add(dep_env_out, format = "--dep_env_out=%s") - args.add(ctx.attr.rundir, format = "--rundir=%s") - output_groups = { "out_dir": depset([out_dir]), } @@ -717,19 +715,6 @@ cargo_build_script = rule( "pkg_name": attr.string( doc = "The name of package being compiled, if not derived from `name`.", ), - "rundir": attr.string( - default = "", - doc = dedent("""\ - A directory to cd to before the cargo_build_script is run. - - This should be a pathrelative to the exec root. The default behaviour (and the - behaviour if rundir is set to the empty string) is to change to the relative - path corresponding to the cargo manifest directory, which replicates the - normal behaviour of cargo so it is easy to write compatible build scripts. - - If set to `.`, the cargo build script will run in the exec root. - """), - ), "rustc_flags": attr.string_list( doc = dedent("""\ List of compiler flags passed to `rustc`. diff --git a/cargo/private/cargo_build_script_runner/bin.rs b/cargo/private/cargo_build_script_runner/bin.rs index 579de21299..79c6afd1a4 100644 --- a/cargo/private/cargo_build_script_runner/bin.rs +++ b/cargo/private/cargo_build_script_runner/bin.rs @@ -18,7 +18,7 @@ use std::collections::BTreeMap; use std::env; use std::fs::{create_dir_all, read_dir, read_to_string, remove_file, write}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use cargo_build_script_runner::cargo_manifest_dir::{remove_symlink, symlink, RunfilesMaker}; @@ -46,7 +46,6 @@ fn run_buildrs() -> Result<(), String> { output_dep_env_path, stdout_path, stderr_path, - rundir, input_dep_env_paths, cargo_manifest_maker, } = Args::parse(); @@ -85,11 +84,9 @@ fn run_buildrs() -> Result<(), String> { let target_env_vars = get_target_env_vars(&rustc_env).expect("Error getting target env vars from rustc"); - let working_directory = resolve_rundir(&rundir, &exec_root, &manifest_dir)?; - let mut command = Command::new(exec_root.join(progname)); command - .current_dir(&working_directory) + .current_dir(&manifest_dir) .envs(target_env_vars) .env("OUT_DIR", &out_dir_abs) .env("CARGO_MANIFEST_DIR", manifest_dir) @@ -320,23 +317,6 @@ fn symlink_if_not_exists(original: &Path, link: &Path) -> Result<(), String> { .map_err(|err| format!("Failed to create symlink: {err}")) } -fn resolve_rundir(rundir: &str, exec_root: &Path, manifest_dir: &Path) -> Result { - if rundir.is_empty() { - return Ok(manifest_dir.to_owned()); - } - let rundir_path = Path::new(rundir); - if rundir_path.is_absolute() { - return Err(format!("rundir must be empty (to run in manifest path) or relative path (relative to exec root), but was {:?}", rundir)); - } - if rundir_path - .components() - .any(|c| c == std::path::Component::ParentDir) - { - return Err(format!("rundir must not contain .. but was {:?}", rundir)); - } - Ok(exec_root.join(rundir_path)) -} - fn swallow_already_exists(err: std::io::Error) -> std::io::Result<()> { if err.kind() == std::io::ErrorKind::AlreadyExists { Ok(()) @@ -357,7 +337,6 @@ struct Args { output_dep_env_path: String, stdout_path: Option, stderr_path: Option, - rundir: String, input_dep_env_paths: Vec, cargo_manifest_maker: RunfilesMaker, } @@ -381,7 +360,6 @@ impl Args { Err("Argument `output_dep_env_path` not provided".to_owned()); let mut stdout_path = None; let mut stderr_path = None; - let mut rundir: Result = Err("Argument `rundir` not provided".to_owned()); let mut input_dep_env_paths = Vec::new(); let mut cargo_manifest_maker: Result = Err("Argument `cargo_manifest_args` not provided".to_owned()); @@ -407,8 +385,6 @@ impl Args { stdout_path = Some(arg.split_off("--stdout=".len())); } else if arg.starts_with("--stderr=") { stderr_path = Some(arg.split_off("--stderr=".len())); - } else if arg.starts_with("--rundir=") { - rundir = Ok(arg.split_off("--rundir=".len())) } else if arg.starts_with("--input_dep_env_path=") { input_dep_env_paths.push(arg.split_off("--input_dep_env_path=".len())); } else if arg.starts_with("--cargo_manifest_args=") { @@ -429,7 +405,6 @@ impl Args { output_dep_env_path: output_dep_env_path.unwrap(), stdout_path, stderr_path, - rundir: rundir.unwrap(), input_dep_env_paths, cargo_manifest_maker: cargo_manifest_maker.unwrap(), } @@ -498,6 +473,7 @@ fn main() { mod test { use super::*; use std::fs::{create_dir_all, write}; + use std::path::PathBuf; fn make_temp_dir(label: &str) -> PathBuf { let nanos = std::time::SystemTime::now() diff --git a/cargo/private/cargo_build_script_wrapper.bzl b/cargo/private/cargo_build_script_wrapper.bzl index bec53cf465..4823a20409 100644 --- a/cargo/private/cargo_build_script_wrapper.bzl +++ b/cargo/private/cargo_build_script_wrapper.bzl @@ -28,7 +28,6 @@ def cargo_build_script( compile_data = [], tools = [], links = None, - rundir = None, rustc_env = {}, rustc_env_files = [], rustc_flags = [], @@ -118,11 +117,6 @@ def cargo_build_script( compile_data (list, optional): Files needed for the compilation of the build script. tools (list, optional): Tools (executables) needed by the build script. links (str, optional): Name of the native library this crate links against. - rundir (str, optional): A directory to `cd` to before the cargo_build_script is run. This should be a path relative to the exec root. - - The default behaviour (and the behaviour if rundir is set to the empty string) is to change to the relative path corresponding to the cargo manifest directory, which replicates the normal behaviour of cargo so it is easy to write compatible build scripts. - - If set to `.`, the cargo build script will run in the exec root. rustc_env (dict, optional): Environment variables to set in rustc when compiling the build script. rustc_env_files (list of label, optional): Files containing additional environment variables to set for rustc when building the build script. @@ -137,6 +131,10 @@ def cargo_build_script( target. """ + rundir = kwargs.pop("rundir", None) + if rundir != None: + fail("cargo_build_script.rundir is no longer supported; build scripts always run in CARGO_MANIFEST_DIR") + # This duplicates the code in _cargo_build_script_impl because we need to make these # available both when we invoke rustc (this code) and when we run the compiled build # script (_cargo_build_script_impl). https://github.com/bazelbuild/rules_rust/issues/661 @@ -225,7 +223,6 @@ def cargo_build_script( links = links, deps = deps, link_deps = link_deps, - rundir = rundir, rustc_flags = rustc_flags, visibility = visibility, tags = tags, diff --git a/cargo/tests/cargo_build_script/run_from_exec_root/BUILD.bazel b/cargo/tests/cargo_build_script/run_from_exec_root/BUILD.bazel deleted file mode 100644 index b0ae7728db..0000000000 --- a/cargo/tests/cargo_build_script/run_from_exec_root/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("//cargo:defs.bzl", "cargo_build_script") -load("//rust:defs.bzl", "rust_test") - -cargo_build_script( - name = "rundir_build_rs", - srcs = ["build.rs"], - data = ["data.txt"], - edition = "2018", - rundir = ".", -) - -rust_test( - name = "test", - srcs = ["test.rs"], - edition = "2018", - deps = [":rundir_build_rs"], -) diff --git a/cargo/tests/cargo_build_script/run_from_exec_root/build.rs b/cargo/tests/cargo_build_script/run_from_exec_root/build.rs deleted file mode 100644 index ee2465dba7..0000000000 --- a/cargo/tests/cargo_build_script/run_from_exec_root/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - let contents = - std::fs::read_to_string("cargo/tests/cargo_build_script/run_from_exec_root/data.txt") - .expect("Failed to read data file"); - println!("cargo:rustc-env=DATA={}", contents); -} diff --git a/cargo/tests/cargo_build_script/run_from_exec_root/data.txt b/cargo/tests/cargo_build_script/run_from_exec_root/data.txt deleted file mode 100644 index e965047ad7..0000000000 --- a/cargo/tests/cargo_build_script/run_from_exec_root/data.txt +++ /dev/null @@ -1 +0,0 @@ -Hello diff --git a/cargo/tests/cargo_build_script/run_from_exec_root/test.rs b/cargo/tests/cargo_build_script/run_from_exec_root/test.rs deleted file mode 100644 index c688fb6a5b..0000000000 --- a/cargo/tests/cargo_build_script/run_from_exec_root/test.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[test] -pub fn test_tool_exec() { - assert_eq!("Hello", env!("DATA")); -} diff --git a/crate_universe/extensions.bzl b/crate_universe/extensions.bzl index aa024737e5..c4610d2334 100644 --- a/crate_universe/extensions.bzl +++ b/crate_universe/extensions.bzl @@ -1360,9 +1360,6 @@ _ANNOTATION_SELECT_ATTRS = { "build_script_proc_macro_deps": _relative_label_list( doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute.", ), - "build_script_rundir": attr.string( - doc = "An override for the build script's rundir attribute.", - ), "build_script_rustc_env": attr.string_dict( doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", ), diff --git a/crate_universe/private/crate.bzl b/crate_universe/private/crate.bzl index e79f8e4a1c..56fece558c 100644 --- a/crate_universe/private/crate.bzl +++ b/crate_universe/private/crate.bzl @@ -99,7 +99,6 @@ def _annotation( build_script_exec_properties = None, build_script_link_deps = None, build_script_proc_macro_deps = None, - build_script_rundir = None, build_script_rustc_env = None, build_script_toolchains = None, build_script_use_default_shell_env = None, @@ -144,7 +143,6 @@ def _annotation( build_script_link_deps: A list of labels to add to a crate's `cargo_build_script::link_deps` attribute. build_script_proc_macro_deps (list, optional): A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute. - build_script_rundir (str, optional): An override for the build script's rundir attribute. build_script_rustc_env (dict, optional): Additional environment variables to set when compiling the crate's `cargo_build_script` - sets that target's `rustc_env` attribute. build_script_toolchains (list, optional): A list of labels to set on a crates's `cargo_build_script::toolchains` attribute. build_script_use_default_shell_env (int, optional): Whether or not to include the default shell environment for the build @@ -207,7 +205,6 @@ def _annotation( build_script_exec_properties = build_script_exec_properties, build_script_link_deps = build_script_link_deps, build_script_proc_macro_deps = _stringify_list(build_script_proc_macro_deps), - build_script_rundir = build_script_rundir, build_script_rustc_env = build_script_rustc_env, build_script_toolchains = _stringify_list(build_script_toolchains), build_script_use_default_shell_env = build_script_use_default_shell_env, diff --git a/crate_universe/src/config.rs b/crate_universe/src/config.rs index 3f6564f7d3..54947e09b0 100644 --- a/crate_universe/src/config.rs +++ b/crate_universe/src/config.rs @@ -338,9 +338,6 @@ pub(crate) struct CrateAnnotations { /// [use_default_shell_env](https://bazelbuild.github.io/rules_rust/cargo.html#cargo_build_script-use_default_shell_env) attribute. pub(crate) build_script_use_default_shell_env: Option, - /// Directory to run the crate's build script in. If not set, will run in the manifest directory, otherwise a directory relative to the exec root. - pub(crate) build_script_rundir: Option>, - /// A scratch pad used to write arbitrary text to target BUILD files. pub(crate) additive_build_file_content: Option, @@ -437,7 +434,6 @@ impl Add for CrateAnnotations { build_script_exec_properties: select_merge(self.build_script_exec_properties, rhs.build_script_exec_properties), build_script_toolchains: joined_extra_member!(self.build_script_toolchains, rhs.build_script_toolchains, BTreeSet::new, BTreeSet::extend), build_script_use_default_shell_env: self.build_script_use_default_shell_env.or(rhs.build_script_use_default_shell_env), - build_script_rundir: self.build_script_rundir.or(rhs.build_script_rundir), additive_build_file_content: joined_extra_member!(self.additive_build_file_content, rhs.additive_build_file_content, String::new, concat_string), shallow_since: self.shallow_since.or(rhs.shallow_since), patch_args: joined_extra_member!(self.patch_args, rhs.patch_args, Vec::new, Vec::extend), @@ -488,7 +484,6 @@ pub(crate) struct AnnotationsProvidedByPackage { pub(crate) rustc_flags: Option>>, pub(crate) build_script_env: Option>>, pub(crate) build_script_rustc_env: Option>>, - pub(crate) build_script_rundir: Option>, pub(crate) additive_build_file_content: Option, pub(crate) extra_aliased_targets: Option>, } @@ -512,7 +507,6 @@ impl CrateAnnotations { rustc_flags, build_script_env, build_script_rustc_env, - build_script_rundir, additive_build_file_content, extra_aliased_targets, } = match AnnotationsProvidedByPackage::deserialize(&pkg_metadata["bazel"]) { @@ -547,7 +541,6 @@ impl CrateAnnotations { default(&mut self.rustc_flags, rustc_flags); default(&mut self.build_script_env, build_script_env); default(&mut self.build_script_rustc_env, build_script_rustc_env); - default(&mut self.build_script_rundir, build_script_rundir); default( &mut self.additive_build_file_content, additive_build_file_content, diff --git a/crate_universe/src/context/crate_context.rs b/crate_universe/src/context/crate_context.rs index 8fdda79dc2..d03b9ad5a6 100644 --- a/crate_universe/src/context/crate_context.rs +++ b/crate_universe/src/context/crate_context.rs @@ -235,9 +235,6 @@ pub(crate) struct BuildScriptAttributes { #[serde(skip_serializing_if = "Select::is_empty")] pub(crate) exec_properties: Select>, - #[serde(skip_serializing_if = "Select::is_empty")] - pub(crate) rundir: Select, - #[serde(skip_serializing_if = "Select::is_empty")] pub(crate) extra_proc_macro_deps: Select>, @@ -284,7 +281,6 @@ impl Default for BuildScriptAttributes { build_script_env: Default::default(), build_script_env_files: Default::default(), exec_properties: Default::default(), - rundir: Default::default(), extra_proc_macro_deps: Default::default(), proc_macro_deps: Default::default(), rustc_env: Default::default(), @@ -714,9 +710,6 @@ impl CrateContext { attrs.use_default_shell_env = Some(*extra); } - if let Some(rundir) = &crate_extra.build_script_rundir { - attrs.rundir = Select::merge(attrs.rundir.clone(), rundir.clone()); - } } // Extra build contents diff --git a/crate_universe/src/lockfile.rs b/crate_universe/src/lockfile.rs index b0d76428ab..fd17c7a6ab 100644 --- a/crate_universe/src/lockfile.rs +++ b/crate_universe/src/lockfile.rs @@ -292,7 +292,7 @@ mod test { ); assert_eq!( - Digest("8a4c1b3bb4c2d6c36e27565e71a13d54cff9490696a492c66a3a37bdd3893edf".to_owned()), + Digest("3aea6884bba019a96aeee55213f930c8210f1028043d1084d2bd49de67e64f3f".to_owned()), digest, ); } diff --git a/crate_universe/src/rendering.rs b/crate_universe/src/rendering.rs index 49360d78ce..401328b8a8 100644 --- a/crate_universe/src/rendering.rs +++ b/crate_universe/src/rendering.rs @@ -21,7 +21,7 @@ use crate::splicing::default_splicing_package_crate_id; use crate::utils::starlark::{ self, Alias, CargoBuildScript, CargoTomlEnvVars, CommonAttrs, Data, ExportsFiles, Filegroup, Glob, Label, Load, Package, RustBinary, RustLibrary, RustProcMacro, SelectDict, SelectList, - SelectScalar, SelectSet, Starlark, TargetCompatibleWith, + SelectSet, Starlark, TargetCompatibleWith, }; use crate::utils::target_triple::TargetTriple; use crate::utils::{self, sanitize_repository_name}; @@ -625,10 +625,6 @@ impl Renderer { ), platforms, ), - rundir: SelectScalar::new( - attrs.map(|attrs| attrs.rundir.clone()).unwrap_or_default(), - platforms, - ), rustc_env: SelectDict::new( attrs .map(|attrs| attrs.rustc_env.clone()) diff --git a/crate_universe/src/utils/starlark.rs b/crate_universe/src/utils/starlark.rs index 14ff5f8c23..b3afe8b94f 100644 --- a/crate_universe/src/utils/starlark.rs +++ b/crate_universe/src/utils/starlark.rs @@ -20,7 +20,6 @@ pub(crate) use label::*; pub(crate) use select::*; pub(crate) use select_dict::*; pub(crate) use select_list::*; -pub(crate) use select_scalar::*; pub(crate) use select_set::*; pub(crate) use target_compatible_with::*; @@ -122,8 +121,6 @@ pub(crate) struct CargoBuildScript { pub(crate) pkg_name: Option, #[serde(skip_serializing_if = "SelectSet::is_empty")] pub(crate) proc_macro_deps: SelectSet