Remove non-deterministic files from OUT_DIR to fix TreeArtifact cache misses

Recursively remove files from OUT_DIR whose names appear in a known list
(config.log, config.log.old, config.status, Makefile, Makefile.config,
config.cache, commit_hash) or have a .d extension before Bazel captures
OUT_DIR as a TreeArtifact.

These files embed sandbox-specific paths, timestamps, or volatile values
that make the TreeArtifact hash non-deterministic, causing cache misses for
all downstream rustc compilations on every action run.

Author: jsharpe

cargo/private/cargo_build_script_runner/bin.rs

@@ -17,7 +17,7 @@
 
 use std::collections::BTreeMap;
 use std::env;
-use std::fs::{create_dir_all, read_to_string, write};
+use std::fs::{create_dir_all, read_dir, read_to_string, remove_file, write};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -238,6 +238,13 @@ fn run_buildrs() -> Result<(), String> {
             .unwrap();
     }
 
+    // Remove non-deterministic configure-generated files from OUT_DIR before
+    // Bazel captures it as a TreeArtifact. Files like config.log and
+    // Makefile.config embed the Bazel sandbox path (which changes on every
+    // action run), making the TreeArtifact hash non-deterministic and causing
+    // cache misses for all downstream rustc compilations.
+    remove_nondeterministic_out_dir_files(&out_dir_abs);
+
     // If out_dir is empty add an empty file to the directory to avoid an upstream Bazel bug
     // https://github.com/bazelbuild/bazel/issues/28286
     if out_dir_abs.read_dir().map(|read| read.count()).unwrap_or(0) == 0 {
@@ -260,6 +267,62 @@ fn run_buildrs() -> Result<(), String> {
     Ok(())
 }
 
+/// OUT_DIR file names that embed the Bazel sandbox path or other volatile
+/// values and must be removed to make the _bs.out_dir TreeArtifact deterministic.
+///
+/// - config.log / config.log.old: autoconf and mklove debug logs.
+/// - config.status: standard autoconf re-run script.
+/// - Makefile: standard autoconf generates this from Makefile.in, embedding
+///   the --prefix (OUT_DIR, sandbox-specific) and CFLAGS. The recursive walk
+///   catches sub-directory Makefiles (e.g. src/Makefile) too.
+/// - Makefile.config: mklove output (rdkafka-sys/librdkafka); contains
+///   CFLAGS/CXXFLAGS/LDFLAGS with sandbox-specific -I/-L paths.
+/// - config.cache: mklove variable cache; contains CFLAGS/LDFLAGS plus
+///   a "generated at $(date)" timestamp header.
+/// - commit_hash: librocksdb-sys writes the git HEAD SHA here to detect
+///   repo-level commit changes; Bazel ignores the resulting
+///   cargo:rerun-if-changed directives but the file changes across commits,
+///   causing cache misses for downstream compilations.
+/// - *.d: Make dependency files generated by gcc/clang; list all included
+///   headers with absolute sandbox paths. rdkafka-sys produces ~91 .d files.
+const NONDETERMINISTIC_OUT_DIR_FILES: &[&str] = &[
+    "config.log",
+    "config.log.old",
+    "config.status",
+    "Makefile",
+    "Makefile.config",
+    "config.cache",
+    "commit_hash",
+];
+
+/// Recursively walk `dir` and delete any file whose name is in
+/// `NONDETERMINISTIC_OUT_DIR_FILES` or has a .d extension (Make dependency files).
+/// Errors are silently ignored: if a file cannot be removed the worst
+/// outcome is a cache miss, not a build failure.
+fn remove_nondeterministic_out_dir_files(dir: &Path) {
+    let entries = match read_dir(dir) {
+        Ok(e) => e,
+        Err(_) => return,
+    };
+    for entry in entries.flatten() {
+        // Use file_type() which does not follow symlinks, so we never recurse
+        // into symlink targets or traverse outside OUT_DIR.
+        let Ok(file_type) = entry.file_type() else {
+            continue;
+        };
+        let path = entry.path();
+        if file_type.is_dir() {
+            remove_nondeterministic_out_dir_files(&path);
+        } else if file_type.is_file() {
+            if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
+                if NONDETERMINISTIC_OUT_DIR_FILES.contains(&name) || name.ends_with(".d") {
+                    let _ = remove_file(&path);
+                }
+            }
+        }
+    }
+}
+
 fn should_symlink_exec_root() -> bool {
     env::var("RULES_RUST_SYMLINK_EXEC_ROOT")
         .map(|s| s == "1")
@@ -449,6 +512,79 @@ fn main() {
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::fs::{create_dir_all, write};
+
+    fn make_temp_dir(label: &str) -> PathBuf {
+        let nanos = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .subsec_nanos();
+        let dir = std::env::temp_dir().join(format!("rules_rust_bin_test_{}_{}", label, nanos));
+        create_dir_all(&dir).unwrap();
+        dir
+    }
+
+    #[test]
+    fn remove_nondeterministic_named_files() {
+        let dir = make_temp_dir("named");
+        for name in NONDETERMINISTIC_OUT_DIR_FILES {
+            write(dir.join(name), "content").unwrap();
+        }
+        write(dir.join("libfoo.a"), "keep").unwrap();
+
+        remove_nondeterministic_out_dir_files(&dir);
+
+        for name in NONDETERMINISTIC_OUT_DIR_FILES {
+            assert!(
+                !dir.join(name).exists(),
+                "{} should have been removed",
+                name
+            );
+        }
+        assert!(dir.join("libfoo.a").exists(), "libfoo.a should be kept");
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn remove_dot_d_files() {
+        let dir = make_temp_dir("dotd");
+        write(dir.join("foo.d"), "deps").unwrap();
+        write(dir.join("bar.d"), "deps").unwrap();
+        write(dir.join("output.o"), "keep").unwrap();
+
+        remove_nondeterministic_out_dir_files(&dir);
+
+        assert!(!dir.join("foo.d").exists(), "foo.d should be removed");
+        assert!(!dir.join("bar.d").exists(), "bar.d should be removed");
+        assert!(dir.join("output.o").exists(), "output.o should be kept");
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn remove_nondeterministic_files_recursively() {
+        let dir = make_temp_dir("recurse");
+        let sub = dir.join("subdir");
+        create_dir_all(&sub).unwrap();
+        write(sub.join("config.log"), "log").unwrap();
+        write(sub.join("foo.d"), "deps").unwrap();
+        write(sub.join("output.o"), "keep").unwrap();
+        write(dir.join("Makefile"), "top-level").unwrap();
+
+        remove_nondeterministic_out_dir_files(&dir);
+
+        assert!(!sub.join("config.log").exists());
+        assert!(!sub.join("foo.d").exists());
+        assert!(sub.join("output.o").exists());
+        assert!(!dir.join("Makefile").exists());
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn remove_nondeterministic_nonexistent_dir_is_noop() {
+        let dir = std::env::temp_dir().join("rules_rust_bin_test_nonexistent_999999999");
+        // Must not panic.
+        remove_nondeterministic_out_dir_files(&dir);
+    }
 
     #[test]
     fn rustc_cfg_parsing() {

cargo/tests/cargo_build_script/nondeterministic_out_dir/BUILD.bazel

@@ -0,0 +1,15 @@
+load("//cargo:defs.bzl", "cargo_build_script")
+load("//rust:defs.bzl", "rust_test")
+
+cargo_build_script(
+    name = "build_script",
+    srcs = ["build.rs"],
+    edition = "2021",
+)
+
+rust_test(
+    name = "test",
+    srcs = ["test.rs"],
+    edition = "2021",
+    deps = [":build_script"],
+)

cargo/tests/cargo_build_script/nondeterministic_out_dir/build.rs

@@ -0,0 +1,29 @@
+use std::path::PathBuf;
+
+fn main() {
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
+
+    // Simulate files written by autoconf/cmake/mklove that embed sandbox-specific
+    // paths and must be stripped by the build script runner before Bazel captures
+    // OUT_DIR as a TreeArtifact.
+    std::fs::write(
+        out_dir.join("config.log"),
+        "configure log with /sandbox/path",
+    )
+    .unwrap();
+    std::fs::write(out_dir.join("config.status"), "configure status").unwrap();
+    std::fs::write(out_dir.join("Makefile"), "all:\n\t@echo sandbox path here").unwrap();
+    std::fs::write(out_dir.join("Makefile.config"), "CFLAGS=-I/sandbox/include").unwrap();
+    std::fs::write(
+        out_dir.join("config.cache"),
+        "# generated at Mon Jan  1 00:00:00 UTC 2024",
+    )
+    .unwrap();
+    std::fs::write(out_dir.join("foo.d"), "foo.o: foo.c /sandbox/include/bar.h").unwrap();
+    std::fs::write(out_dir.join("baz.d"), "baz.o: baz.c /sandbox/include/qux.h").unwrap();
+
+    // Write a legitimate output that downstream consumers must be able to read.
+    std::fs::write(out_dir.join("output.txt"), "legitimate output").unwrap();
+
+    println!("cargo:rerun-if-changed=build.rs");
+}

cargo/tests/cargo_build_script/nondeterministic_out_dir/test.rs

@@ -0,0 +1,10 @@
+// include_str! resolves at compile time against the TreeArtifact captured by Bazel.
+// If the runner had removed output.txt this would not compile; if it had failed to
+// strip config.log / *.d files, the TreeArtifact hash would change on every run,
+// causing unnecessary rebuilds for all downstream crates.
+const OUTPUT: &str = include_str!(concat!(env!("OUT_DIR"), "/output.txt"));
+
+#[test]
+fn legitimate_output_survives_nondeterministic_file_removal() {
+    assert_eq!(OUTPUT, "legitimate output");
+}