Computing crate_hash from metadata encoding instead of HIR (implements #94878) (very draft)

compiler/rustc_ast_lowering/src/lib.rs

@@ -45,7 +45,7 @@ use rustc_attr_parsing::{AttributeParser, OmitDoc, Recovery, ShouldEmit};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::sorted_map::SortedMap;
-use rustc_data_structures::stable_hasher::{StableHash, StableHasher};
+//use rustc_data_structures::stable_hasher::{StableHash, StableHasher};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::tagged_ptr::TaggedRef;
 use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle};
@@ -510,21 +510,16 @@ fn compute_hir_hash(
     tcx: TyCtxt<'_>,
     owners: &IndexSlice<LocalDefId, hir::MaybeOwner<'_>>,
 ) -> Fingerprint {
-    let mut hir_body_nodes: Vec<_> = owners
+    owners
         .iter_enumerated()
         .filter_map(|(def_id, info)| {
             let info = info.as_owner()?;
-            let def_path_hash = tcx.hir_def_path_hash(def_id);
-            Some((def_path_hash, info))
+            let dph = tcx.hir_def_path_hash(def_id).0;
+            let info_finger = info.fingerprint();
+            Some(dph.combine(info_finger))
         })
-        .collect();
-    hir_body_nodes.sort_unstable_by_key(|bn| bn.0);
-
-    tcx.with_stable_hashing_context(|mut hcx| {
-        let mut stable_hasher = StableHasher::new();
-        hir_body_nodes.stable_hash(&mut hcx, &mut stable_hasher);
-        stable_hasher.finish()
-    })
+        .reduce(Fingerprint::combine_commutative)
+        .expect("HIR hash requested without any content")
 }
 
 pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> mid_hir::Crate<'_> {

compiler/rustc_driver_impl/src/lib.rs

@@ -324,10 +324,6 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 
             tcx.ensure_ok().analysis(());
 
-            if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir {
-                dump_feature_usage_metrics(tcx, metrics_dir);
-            }
-
             if callbacks.after_analysis(compiler, tcx) == Compilation::Stop {
                 return early_exit();
             }
@@ -340,6 +336,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 
             let linker = Linker::codegen_and_build_linker(tcx, &*compiler.codegen_backend);
 
+            if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir {
+                dump_feature_usage_metrics(tcx, metrics_dir);
+            }
+
             tcx.report_unused_features();
 
             Some(linker)

compiler/rustc_hir/src/hir.rs

@@ -1645,6 +1645,16 @@ impl<'tcx> OwnerInfo<'tcx> {
     pub fn node(&self) -> OwnerNode<'tcx> {
         self.nodes.node()
     }
+
+    #[inline]
+    pub fn fingerprint(&self) -> Fingerprint {
+        let body = self
+            .nodes
+            .opt_hash_including_bodies
+            .expect("HIR hash requested without needs_hir_hash");
+        let attrs = self.attrs.opt_hash.expect("HIR hash requested without needs_hir_hash");
+        body.combine(attrs)
+    }
 }
 
 #[derive(Copy, Clone, Debug, StableHash)]

compiler/rustc_interface/src/passes.rs

@@ -948,8 +948,13 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
     let definitions = FreezeLock::new(Definitions::new(stable_crate_id));
 
     let stable_crate_ids = FreezeLock::new(StableCrateIdMap::default());
-    let untracked =
-        Untracked { cstore, source_span: AppendOnlyIndexVec::new(), definitions, stable_crate_ids };
+    let untracked = Untracked {
+        cstore,
+        source_span: AppendOnlyIndexVec::new(),
+        definitions,
+        stable_crate_ids,
+        local_crate_hash: OnceLock::new(),
+    };
 
     // We're constructing the HIR here; we don't care what we will
     // read, since we haven't even constructed the *input* to

compiler/rustc_metadata/src/lib.rs

@@ -1,5 +1,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![feature(core_intrinsics)]
 #![feature(error_iter)]
 #![feature(file_buffered)]
 #![feature(gen_blocks)]

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -2,13 +2,17 @@ use std::any::Any;
 use std::mem;
 use std::sync::Arc;
 
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::stable_hasher::{StableHash, StableHasher};
+use rustc_data_structures::svh::Svh;
 use rustc_hir::attrs::Deprecation;
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash};
 use rustc_middle::arena::ArenaAllocatable;
 use rustc_middle::bug;
 use rustc_middle::metadata::{AmbigModChild, ModChild};
+//use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
 use rustc_middle::middle::exported_symbols::ExportedSymbol;
 use rustc_middle::middle::stability::DeprecationEntry;
 use rustc_middle::queries::ExternProviders;
@@ -750,4 +754,104 @@ fn provide_cstore_hooks(providers: &mut Providers) {
             cdata.imported_source_file(tcx, file_index as u32);
         }
     };
+
+    providers.queries.crate_hash = |tcx: TyCtxt<'_>, _: LocalCrate| {
+        if tcx.needs_metadata() {
+            *tcx.untracked()
+                .local_crate_hash
+                .get()
+                .expect("crate_hash(LOCAL_CRATE) called before metadata encoding")
+        } else {
+            crate_hash(tcx)
+        }
+    };
 }
+
+pub(super) fn crate_hash(tcx: TyCtxt<'_>) -> Svh {
+    let krate = tcx.hir_crate(());
+    let hir_body_hash = krate.opt_hir_hash.expect("HIR hash missing while computing crate hash");
+
+    let upstream_crates = tcx.crates(()); //upstream_crates(tcx);
+
+    // let resolutions = tcx.resolutions(());
+
+    // We hash the final, remapped names of all local source files so we
+    // don't have to include the path prefix remapping commandline args.
+    // If we included the full mapping in the SVH, we could only have
+    // reproducible builds by compiling from the same directory. So we just
+    // hash the result of the mapping instead of the mapping itself.
+    /*let mut source_file_names: Vec<_> = tcx
+        .sess
+        .source_map()
+        .files()
+        .iter()
+        .filter(|source_file| source_file.cnum == LOCAL_CRATE)
+        .map(|source_file| source_file.stable_id)
+        .collect();
+
+    source_file_names.sort_unstable();*/
+
+    // We have to take care of debugger visualizers explicitly. The HIR (and
+    // thus `hir_body_hash`) contains the #[debugger_visualizer] attributes but
+    // these attributes only store the file path to the visualizer file, not
+    // their content. Yet that content is exported into crate metadata, so any
+    // changes to it need to be reflected in the crate hash.
+    /*let debugger_visualizers: Vec<_> = tcx
+    .debugger_visualizers(LOCAL_CRATE)
+    .iter()
+    // We ignore the path to the visualizer file since it's not going to be
+    // encoded in crate metadata and we already hash the full contents of
+    // the file.
+    .map(DebuggerVisualizerFile::path_erased)
+    .collect();*/
+
+    let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| {
+        let mut stable_hasher = StableHasher::new();
+        hir_body_hash.stable_hash(&mut hcx, &mut stable_hasher);
+        upstream_crates.stable_hash(&mut hcx, &mut stable_hasher);
+        //source_file_names.stable_hash(&mut hcx, &mut stable_hasher);
+        //debugger_visualizers.stable_hash(&mut hcx, &mut stable_hasher);
+        /*if tcx.sess.opts.incremental.is_some() {
+            let definitions = tcx.untracked().definitions.freeze();
+            let mut owner_spans: Vec<_> = tcx
+                .hir_crate_items(())
+                .definitions()
+                .map(|def_id| {
+                    let def_path_hash = definitions.def_path_hash(def_id);
+                    let span = tcx.source_span(def_id);
+                    debug_assert_eq!(span.parent(), None);
+                    (def_path_hash, span)
+                })
+                .collect();
+            owner_spans.sort_unstable_by_key(|bn| bn.0);
+            owner_spans.stable_hash(&mut hcx, &mut stable_hasher);
+        }*/
+        tcx.sess.opts.dep_tracking_hash(true).stable_hash(&mut hcx, &mut stable_hasher);
+        tcx.stable_crate_id(LOCAL_CRATE).stable_hash(&mut hcx, &mut stable_hasher);
+        // Hash visibility information since it does not appear in HIR.
+        // FIXME: Figure out how to remove `visibilities_for_hashing` by hashing visibilities on
+        // the fly in the resolver, storing only their accumulated hash in `ResolverGlobalCtxt`,
+        // and combining it with other hashes here.
+        //resolutions.visibilities_for_hashing.stable_hash(&mut hcx, &mut stable_hasher);
+        /*with_metavar_spans(|mspans| {
+            mspans.freeze_and_get_read_spans().stable_hash(&mut hcx, &mut stable_hasher);
+        });*/
+        stable_hasher.finish()
+    });
+
+    Svh::new(crate_hash)
+}
+
+/*fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
+    let mut upstream_crates: Vec<_> = tcx
+        .crates(())
+        .iter()
+        .map(|&cnum| {
+            let stable_crate_id = tcx.stable_crate_id(cnum);
+            let hash = tcx.crate_hash(cnum);
+            (stable_crate_id, hash)
+        })
+        .collect();
+    upstream_crates.sort_unstable_by_key(|&(stable_crate_id, _)| stable_crate_id);
+    upstream_crates
+}*/