fix: probe flag support without OUT_DIR via tempfile (#1875)

Author: cestercian

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Probe flag support without `OUT_DIR` via tempfile, so `flag_if_supported` no longer silently drops flags outside Cargo build scripts ([#1875](https://github.com/rust-lang/cc-rs/pull/1875))
+
 ## [1.4.4](https://github.com/rust-lang/cc-rs/compare/cc-v1.4.3...cc-v1.4.4) - 2026-08-21
 
 ### Fixed

src/lib.rs

@@ -1445,6 +1445,19 @@ impl Build {
     }
 }
 
+/// Source, object, and working directory for an `is_flag_supported` probe.
+///
+/// Tempfiles, when used, are removed when this value is dropped.
+struct FlagSupportProbeFiles<'a> {
+    dir: Cow<'a, Path>,
+    src: PathBuf,
+    obj: PathBuf,
+    _temp_files: Option<(
+        crate::tempfile::NamedTempfile,
+        crate::tempfile::NamedTempfile,
+    )>,
+}
+
 /// Invoke or fetch the compiler or archiver.
 impl Build {
     /// Run the compiler to test if it accepts the given flag.
@@ -1466,25 +1479,81 @@ impl Build {
         )
     }
 
-    fn ensure_check_file(&self) -> Result<PathBuf, Error> {
-        let out_dir = self.get_out_dir()?;
-        let src = if self.cuda {
+    fn flag_check_src_name(&self) -> &'static str {
+        if self.cuda {
             assert!(self.cpp);
-            out_dir.join("flag_check.cu")
+            "flag_check.cu"
         } else if self.cpp {
-            out_dir.join("flag_check.cpp")
+            "flag_check.cpp"
         } else {
-            out_dir.join("flag_check.c")
-        };
+            "flag_check.c"
+        }
+    }
+
+    fn write_flag_check_src(file: &mut fs::File) -> io::Result<()> {
+        write!(file, "int main(void) {{ return 0; }}")?;
+        file.flush()?;
+        file.sync_data()
+    }
+
+    fn ensure_check_file(&self) -> Result<PathBuf, Error> {
+        let src = self.get_out_dir()?.join(self.flag_check_src_name());
 
         if !src.exists() {
             let mut f = fs::File::create(&src)?;
-            write!(f, "int main(void) {{ return 0; }}")?;
+            Self::write_flag_check_src(&mut f)?;
         }
 
         Ok(src)
     }
 
+    /// Directory, source, and object for a flag-support probe.
+    ///
+    /// Cargo build scripts have `OUT_DIR`; reuse `flag_check.c` there so
+    /// probes stay cheap. Callers such as rustc bootstrap do not, and
+    /// treating a missing dir as "unsupported" silently drops flags.
+    /// Fall back to unique tempfiles instead of a shared name in `/tmp`.
+    fn flag_support_probe_files(&self) -> Result<FlagSupportProbeFiles<'_>, Error> {
+        match self.get_out_dir() {
+            Ok(dir) => {
+                let src = self.ensure_check_file()?;
+                let obj = dir.join("flag_check");
+                Ok(FlagSupportProbeFiles {
+                    dir,
+                    src,
+                    obj,
+                    _temp_files: None,
+                })
+            }
+            Err(_) => {
+                let dir = env::temp_dir();
+                fs::create_dir_all(&dir)?;
+
+                let mut tmp_src =
+                    crate::tempfile::NamedTempfile::new(&dir, self.flag_check_src_name())?;
+                let mut tmp_file = tmp_src.take_file().unwrap();
+                Self::write_flag_check_src(&mut tmp_file)?;
+                // Close the handle before invoking the compiler; Windows
+                // cannot open a file that another handle still holds.
+                drop(tmp_file);
+
+                let mut tmp_obj = crate::tempfile::NamedTempfile::new(&dir, "flag_check")?;
+                // Same as the source file: the compiler must be able to
+                // overwrite this path, so drop the open handle first.
+                drop(tmp_obj.take_file());
+
+                let src = tmp_src.path().to_owned();
+                let obj = tmp_obj.path().to_owned();
+                Ok(FlagSupportProbeFiles {
+                    dir: Cow::Owned(dir),
+                    src,
+                    obj,
+                    _temp_files: Some((tmp_src, tmp_obj)),
+                })
+            }
+        }
+    }
+
     fn is_flag_supported_inner(
         &self,
         flag: &OsStr,
@@ -1507,9 +1576,7 @@ impl Build {
             return Ok(is_supported);
         }
 
-        let out_dir = self.get_out_dir()?;
-        let src = self.ensure_check_file()?;
-        let obj = out_dir.join("flag_check");
+        let probe = self.flag_support_probe_files()?;
 
         let mut compiler = {
             let mut cfg = Build::new();
@@ -1520,7 +1587,7 @@ impl Build {
                 .debug(false)
                 .cpp(self.cpp)
                 .cuda(self.cuda)
-                .out_dir(&out_dir)
+                .out_dir(&*probe.dir)
                 .inherit_rustflags(false)
                 .inherit_trim_paths(false)
                 .emit_rerun_if_env_changed(self.emit_rerun_if_env_changed);
@@ -1555,7 +1622,7 @@ impl Build {
         cmd.set_flag_supported_env(&self.env);
         command_add_output_file(
             &mut cmd,
-            &obj,
+            &probe.obj,
             CmdAddOutputFileArgs {
                 cuda: self.cuda,
                 is_assembler_msvc: false,
@@ -1577,7 +1644,7 @@ impl Build {
             cmd.arg("--");
         }
 
-        cmd.arg(&src);
+        cmd.arg(&probe.src);
 
         if compiler.is_like_msvc() {
             // On MSVC we need to make sure the LIB directory is included
@@ -1590,10 +1657,11 @@ impl Build {
             }
         }
 
-        cmd.current_dir(out_dir);
+        cmd.current_dir(&*probe.dir);
         self.cargo_output
             .print_debug(&format_args!("running: {cmd:?}"));
         let output = cmd.output()?;
+        drop(probe);
         let is_supported = output.status.success() && output.stderr.is_empty();
 
         self.build_cache

tests/support/mod.rs

@@ -125,6 +125,16 @@ impl Test {
     }
 
     pub fn gcc(&self) -> cc::Build {
+        let mut cfg = self.gcc_without_out_dir();
+        cfg.out_dir(self.td.path());
+        cfg
+    }
+
+    /// Like [`Self::gcc`], but does not set [`cc::Build::out_dir`].
+    ///
+    /// Flag-support probes must still work when `OUT_DIR` is unset, as in
+    /// rustc bootstrap which is not a Cargo build script.
+    pub fn gcc_without_out_dir(&self) -> cc::Build {
         let mut cfg = cc::Build::new();
         let target = if self.msvc || self.msvc_autodetect {
             "x86_64-pc-windows-msvc"
@@ -138,7 +148,6 @@ impl Test {
             .host(target)
             .opt_level(2)
             .debug(false)
-            .out_dir(self.td.path())
             .env("PATH", self.path())
             .env("CC_SHIM_OUT_DIR", self.td.path());
         if self.family_detection_probes {

tests/test.rs

@@ -415,6 +415,38 @@ fn gnu_flag_if_supported() {
         .must_not_have("-Wflag-does-not-exist");
 }
 
+/// `flag_if_supported` must probe even when `OUT_DIR` is unset.
+/// <https://github.com/rust-lang/cc-rs/issues/1765>
+#[test]
+fn flag_if_supported_without_out_dir() {
+    let mut test = Test::gnu();
+    test.env.remove("OUT_DIR");
+    test.collect_flag_supported_probes();
+
+    let compiler = test
+        .gcc_without_out_dir()
+        .env("CC_SHIM_FAIL_IF_ARG", "-Wflag-does-not-exist")
+        .flag_if_supported("-Wall")
+        .flag_if_supported("-Wflag-does-not-exist")
+        .try_get_compiler()
+        .expect("try_get_compiler should succeed without OUT_DIR");
+
+    assert!(
+        compiler.args().iter().any(|a| a == "-Wall"),
+        "supported flag should be applied without OUT_DIR, args: {:?}",
+        compiler.args()
+    );
+    assert!(
+        !compiler.args().iter().any(|a| a == "-Wflag-does-not-exist"),
+        "unsupported flag should still be rejected without OUT_DIR, args: {:?}",
+        compiler.args()
+    );
+
+    test.get_flag_supported_probes(0)
+        .must_have("-Wall")
+        .must_have("-c");
+}
+
 /// cc's own probing invocations run in the environment `Build::env` sets up,
 /// and record only the class of probe a test asks for by name.
 /// <https://github.com/rust-lang/cc-rs/issues/1859>