Skip to content

Commit 4b2fbe8

Browse files
DaniPopessylvestre
authored andcommitted
Calculate rustc outputs concurrently with other hashing tasks
Currently, `sccache` invokes `rustc` twice for generating the hash key: once for the inputs and env-deps (`--emit dep-info`), and another time for the output file names (`--print file-names`). However, the latter is invoked only after the former finishes. There is no reason for this dependency. Run `get_compiler_outputs` concurrently with the other `rustc` and hashing tasks in `RustHasher::generate_hash_key`. I initially thought that these two invocations could be merged together, but `--print` stop compilation before the `--emit` is evaluated, so I went with the simpler approach of just running them at the same time.
1 parent d9d2eb6 commit 4b2fbe8

1 file changed

Lines changed: 29 additions & 25 deletions

File tree

src/compiler/rust.rs

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ static ALLOWED_EMIT: LazyLock<HashSet<&'static str>> =
237237
/// Version number for cache key.
238238
const CACHE_VERSION: &[u8] = b"6";
239239

240-
/// Get absolute paths for all source files and env-deps listed in rustc's dep-info output.
240+
/// Run `rustc --emit=dep-info` to get the list of source files and env-deps.
241241
async fn get_source_files_and_env_deps<T>(
242242
creator: &T,
243243
crate_name: &str,
@@ -381,22 +381,25 @@ async fn get_compiler_outputs<T>(
381381
where
382382
T: Clone + CommandCreatorSync,
383383
{
384+
let start = time::Instant::now();
384385
let mut cmd = creator.clone().new_command_sync(executable);
385386
cmd.args(&arguments)
386387
.args(&["--print", "file-names"])
387388
.env_clear()
388389
.envs(env_vars.to_vec())
389390
.current_dir(cwd);
390-
if log_enabled!(Trace) {
391-
trace!("get_compiler_outputs: {:?}", cmd);
392-
}
391+
trace!("get_compiler_outputs: {:?}", cmd);
393392
let outputs = run_input_output(cmd, None).await?;
394393

395394
let outstr = String::from_utf8(outputs.stdout).context("Error parsing rustc output")?;
396-
if log_enabled!(Trace) {
397-
trace!("get_compiler_outputs: {:?}", outstr);
398-
}
399-
Ok(outstr.lines().map(|l| l.to_owned()).collect())
395+
let outputs = outstr.lines().map(|l| l.to_owned()).collect::<Vec<_>>();
396+
trace!(
397+
"get_compiler_outputs: got {} outputs in {}: {:?}",
398+
outputs.len(),
399+
fmt_duration_as_secs(&start.elapsed()),
400+
outstr,
401+
);
402+
Ok(outputs)
400403
}
401404

402405
impl Rust {
@@ -1381,6 +1384,20 @@ where
13811384
Ok((source_files, source_hashes, env_deps))
13821385
};
13831386

1387+
// Turn arguments into a simple Vec<OsString> to calculate outputs.
1388+
let flat_os_string_arguments: Vec<OsString> = os_string_arguments
1389+
.iter()
1390+
.cloned()
1391+
.flat_map(|(arg, val)| iter::once(arg).chain(val))
1392+
.collect();
1393+
let outputs = get_compiler_outputs(
1394+
creator,
1395+
&self.executable,
1396+
flat_os_string_arguments,
1397+
&cwd,
1398+
&env_vars,
1399+
);
1400+
13841401
// Hash the contents of the externs listed on the commandline.
13851402
trace!(
13861403
"[{}]: hashing {} externs",
@@ -1422,17 +1439,19 @@ where
14221439

14231440
let target_json_hash = hash_all(&target_json_files, pool);
14241441

1425-
// Perform all hashing operations on the files.
1442+
// Invoke the compiler and perform all hashing operations on the files.
14261443
let (
14271444
(source_files, source_hashes, mut env_deps),
14281445
extern_hashes,
14291446
staticlib_hashes,
14301447
target_json_hash,
1448+
mut outputs,
14311449
) = futures::try_join!(
14321450
source_files_and_hashes_and_env_deps,
14331451
extern_hashes,
14341452
staticlib_hashes,
1435-
target_json_hash
1453+
target_json_hash,
1454+
outputs,
14361455
)?;
14371456

14381457
// If you change any of the inputs to the hash, you should change `CACHE_VERSION`.
@@ -1539,21 +1558,6 @@ where
15391558
// 10. The version of the compiler.
15401559
self.version.hash(&mut HashToDigest { digest: &mut m });
15411560

1542-
// Turn arguments into a simple Vec<OsString> to calculate outputs.
1543-
let flat_os_string_arguments: Vec<OsString> = os_string_arguments
1544-
.into_iter()
1545-
.flat_map(|(arg, val)| iter::once(arg).chain(val))
1546-
.collect();
1547-
1548-
let mut outputs = get_compiler_outputs(
1549-
creator,
1550-
&self.executable,
1551-
flat_os_string_arguments,
1552-
&cwd,
1553-
&env_vars,
1554-
)
1555-
.await?;
1556-
15571561
// metadata / dep-info don't ever generate binaries, but
15581562
// rustc still makes them appear in the --print
15591563
// file-names output (see

0 commit comments

Comments
 (0)