Experiment: No dynamic dispatch for query functions

Cargo.lock

@@ -4169,7 +4169,6 @@ dependencies = [
  "rustc_parse",
  "rustc_passes",
  "rustc_privacy",
- "rustc_query_impl",
  "rustc_resolve",
  "rustc_session",
  "rustc_span",
@@ -4303,6 +4302,7 @@ dependencies = [
  "bitflags",
  "either",
  "gsgdt",
+ "measureme",
  "parking_lot",
  "polonius-engine",
  "rustc_abi",
@@ -4564,23 +4564,6 @@ dependencies = [
  "rustc_target",
 ]
 
-[[package]]
-name = "rustc_query_impl"
-version = "0.0.0"
-dependencies = [
- "measureme",
- "rustc_abi",
- "rustc_data_structures",
- "rustc_errors",
- "rustc_hir",
- "rustc_macros",
- "rustc_middle",
- "rustc_serialize",
- "rustc_span",
- "rustc_thread_pool",
- "tracing",
-]
-
 [[package]]
 name = "rustc_resolve"
 version = "0.0.0"

compiler/rustc_codegen_llvm/src/base.rs

@@ -22,6 +22,7 @@ use rustc_hir::attrs::Linkage;
 use rustc_middle::dep_graph;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs};
 use rustc_middle::mono::Visibility;
+use rustc_middle::query::{QueryHashHelper, QueryHelper};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{DebugInfo, Offload};
 use rustc_span::Symbol;
@@ -62,122 +63,175 @@ pub(crate) fn compile_codegen_unit(
     let start_time = Instant::now();
 
     let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx);
-    let (module, _) = tcx.dep_graph.with_task(
-        dep_node,
-        tcx,
-        cgu_name,
-        module_codegen,
-        Some(dep_graph::hash_result),
-    );
-    let time_to_codegen = start_time.elapsed();
 
-    // We assume that the cost to run LLVM on a CGU is proportional to
-    // the time we needed for codegenning it.
-    let cost = time_to_codegen.as_nanos() as u64;
+    #[derive(Default)]
+    struct CompileCguHashHelper;
 
-    fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm> {
-        let cgu = tcx.codegen_unit(cgu_name);
-        let _prof_timer =
-            tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| {
-                recorder.record_arg(cgu_name.to_string());
-                recorder.record_arg(cgu.size_estimate().to_string());
-            });
-        // Instantiate monomorphizations without filling out definitions yet...
-        let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str());
-        {
-            let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
-
-            // Declare and store globals shared by all offload kernels
-            //
-            // These globals are left in the LLVM-IR host module so all kernels can access them.
-            // They are necessary for correct offload execution. We do this here to simplify the
-            // `offload` intrinsic, avoiding the need for tracking whether it's the first
-            // intrinsic call or not.
-            let has_host_offload = cx
-                .sess()
-                .opts
-                .unstable_opts
-                .offload
-                .iter()
-                .any(|o| matches!(o, Offload::Host(_) | Offload::Test));
-            if has_host_offload && !cx.sess().target.is_like_gpu {
-                cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx)));
-            }
+    impl QueryHashHelper<ModuleCodegen<ModuleLlvm>> for CompileCguHashHelper {
+        const NO_HASH: bool = false;
 
-            let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
-            for &(mono_item, data) in &mono_items {
-                mono_item.predefine::<Builder<'_, '_, '_>>(
-                    &mut cx,
-                    cgu_name.as_str(),
-                    data.linkage,
-                    data.visibility,
-                );
-            }
+        fn hash_value_fn(
+            hcx: &mut rustc_middle::ich::StableHashingContext<'_>,
+            result: &ModuleCodegen<ModuleLlvm>,
+        ) -> rustc_data_structures::fingerprint::Fingerprint {
+            dep_graph::hash_result(hcx, result)
+        }
 
-            // ... and now that we have everything pre-defined, fill out those definitions.
-            for &(mono_item, item_data) in &mono_items {
-                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
-            }
+        #[cold]
+        fn format_value(_: &ModuleCodegen<ModuleLlvm>) -> String {
+            unimplemented!()
+        }
+    }
+
+    // FIXME: use a separate trait for this
+    impl<'tcx> QueryHelper<'tcx, Symbol, ModuleCodegen<ModuleLlvm>> for CompileCguHashHelper {
+        const EVAL_ALWAYS: bool = false;
+        const DEPTH_LIMIT: bool = false;
+        const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::CompileCodegenUnit;
+        const FEEDABLE: bool = false;
+        const NAME: &'static str = "";
 
-            // If this codegen unit contains the main function, also create the
-            // wrapper here
-            if let Some(entry) =
-                maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
+        #[cold]
+        fn try_load_from_disk_fn(
+            _: TyCtxt<'tcx>,
+            _: dep_graph::SerializedDepNodeIndex,
+        ) -> Option<ModuleCodegen<ModuleLlvm>> {
+            unimplemented!()
+        }
+
+        #[cold]
+        fn will_cache_on_disk_for_key(_: Symbol) -> bool {
+            unimplemented!()
+        }
+
+        fn invoke_provider_fn(tcx: TyCtxt<'tcx>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm> {
+            // Module codegen
+            let cgu = tcx.codegen_unit(cgu_name);
+            let _prof_timer =
+                tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| {
+                    recorder.record_arg(cgu_name.to_string());
+                    recorder.record_arg(cgu.size_estimate().to_string());
+                });
+            // Instantiate monomorphizations without filling out definitions yet...
+            let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str());
             {
-                let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default());
-                attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
-            }
+                let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
 
-            // Define Objective-C module info and module flags. Note, the module info will
-            // also be added to the `llvm.compiler.used` variable, created later.
-            //
-            // These are only necessary when we need the linker to do its Objective-C-specific
-            // magic. We could theoretically do it unconditionally, but at a slight cost to linker
-            // performance in the common case where it's unnecessary.
-            if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() {
-                if cx.objc_abi_version() == 1 {
-                    cx.define_objc_module_info();
+                // Declare and store globals shared by all offload kernels
+                //
+                // These globals are left in the LLVM-IR host module so all kernels can access them.
+                // They are necessary for correct offload execution. We do this here to simplify the
+                // `offload` intrinsic, avoiding the need for tracking whether it's the first
+                // intrinsic call or not.
+                let has_host_offload = cx
+                    .sess()
+                    .opts
+                    .unstable_opts
+                    .offload
+                    .iter()
+                    .any(|o| matches!(o, Offload::Host(_) | Offload::Test));
+                if has_host_offload && !cx.sess().target.is_like_gpu {
+                    cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx)));
                 }
-                cx.add_objc_module_flags();
-            }
 
-            // Finalize code coverage by injecting the coverage map. Note, the coverage map will
-            // also be added to the `llvm.compiler.used` variable, created next.
-            if cx.sess().instrument_coverage() {
-                cx.coverageinfo_finalize();
-            }
+                let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
+                for &(mono_item, data) in &mono_items {
+                    mono_item.predefine::<Builder<'_, '_, '_>>(
+                        &mut cx,
+                        cgu_name.as_str(),
+                        data.linkage,
+                        data.visibility,
+                    );
+                }
 
-            // Create the llvm.used variable.
-            if !cx.used_statics.is_empty() {
-                cx.create_used_variable_impl(c"llvm.used", &cx.used_statics);
-            }
+                // ... and now that we have everything pre-defined, fill out those definitions.
+                for &(mono_item, item_data) in &mono_items {
+                    mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
+                }
 
-            // Create the llvm.compiler.used variable.
-            {
-                let compiler_used_statics = cx.compiler_used_statics.borrow();
-                if !compiler_used_statics.is_empty() {
-                    cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics);
+                // If this codegen unit contains the main function, also create the
+                // wrapper here
+                if let Some(entry) =
+                    maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
+                {
+                    let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default());
+                    attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
                 }
-            }
 
-            // Run replace-all-uses-with for statics that need it. This must
-            // happen after the llvm.used variables are created.
-            for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() {
-                unsafe {
-                    llvm::LLVMReplaceAllUsesWith(old_g, new_g);
-                    llvm::LLVMDeleteGlobal(old_g);
+                // Define Objective-C module info and module flags. Note, the module info will
+                // also be added to the `llvm.compiler.used` variable, created later.
+                //
+                // These are only necessary when we need the linker to do its Objective-C-specific
+                // magic. We could theoretically do it unconditionally, but at a slight cost to linker
+                // performance in the common case where it's unnecessary.
+                if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() {
+                    if cx.objc_abi_version() == 1 {
+                        cx.define_objc_module_info();
+                    }
+                    cx.add_objc_module_flags();
+                }
+
+                // Finalize code coverage by injecting the coverage map. Note, the coverage map will
+                // also be added to the `llvm.compiler.used` variable, created next.
+                if cx.sess().instrument_coverage() {
+                    cx.coverageinfo_finalize();
+                }
+
+                // Create the llvm.used variable.
+                if !cx.used_statics.is_empty() {
+                    cx.create_used_variable_impl(c"llvm.used", &cx.used_statics);
                 }
-            }
 
-            // Finalize debuginfo
-            if cx.sess().opts.debuginfo != DebugInfo::None {
-                cx.debuginfo_finalize();
+                // Create the llvm.compiler.used variable.
+                {
+                    let compiler_used_statics = cx.compiler_used_statics.borrow();
+                    if !compiler_used_statics.is_empty() {
+                        cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics);
+                    }
+                }
+
+                // Run replace-all-uses-with for statics that need it. This must
+                // happen after the llvm.used variables are created.
+                for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() {
+                    unsafe {
+                        llvm::LLVMReplaceAllUsesWith(old_g, new_g);
+                        llvm::LLVMDeleteGlobal(old_g);
+                    }
+                }
+
+                // Finalize debuginfo
+                if cx.sess().opts.debuginfo != DebugInfo::None {
+                    cx.debuginfo_finalize();
+                }
             }
+
+            ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module)
+        }
+
+        #[cold]
+        fn create_tagged_key(_: Symbol) -> rustc_middle::queries::TaggedQueryKey<'tcx> {
+            unimplemented!()
         }
 
-        ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module)
+        #[cold]
+        fn handle_cycle_error_fn(
+            _: TyCtxt<'tcx>,
+            _: Symbol,
+            _: rustc_middle::query::Cycle<'tcx>,
+            _: rustc_errors::Diag<'_>,
+        ) -> ModuleCodegen<ModuleLlvm> {
+            unimplemented!()
+        }
     }
 
+    let (module, _) =
+        tcx.dep_graph.with_task::<_, _, CompileCguHashHelper>(dep_node, tcx, cgu_name);
+    let time_to_codegen = start_time.elapsed();
+
+    // We assume that the cost to run LLVM on a CGU is proportional to
+    // the time we needed for codegenning it.
+    let cost = time_to_codegen.as_nanos() as u64;
+
     (module, cost)
 }
 

compiler/rustc_interface/Cargo.toml

@@ -34,7 +34,6 @@ rustc_monomorphize = { path = "../rustc_monomorphize" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_passes = { path = "../rustc_passes" }
 rustc_privacy = { path = "../rustc_privacy" }
-rustc_query_impl = { path = "../rustc_query_impl" }
 rustc_resolve = { path = "../rustc_resolve" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }

compiler/rustc_interface/src/interface.rs

@@ -9,14 +9,14 @@ use rustc_data_structures::jobserver::{self, Proxy};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed};
 use rustc_lint::LintStore;
+use rustc_middle::query::impl_::print_query_stack;
 use rustc_middle::ty;
 use rustc_middle::ty::CurrentGcx;
 use rustc_middle::util::Providers;
 use rustc_parse::lexer::StripTokens;
 use rustc_parse::new_parser_from_source_str;
 use rustc_parse::parser::Recovery;
 use rustc_parse::parser::attr::AllowLeadingUnsafe;
-use rustc_query_impl::print_query_stack;
 use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
 use rustc_session::parse::ParseSess;
 use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint};

compiler/rustc_interface/src/passes.rs

@@ -787,7 +787,7 @@ fn resolver_for_lowering_raw<'tcx>(
 ) -> (&'tcx Steal<(ty::ResolverAstLowering<'tcx>, Arc<ast::Crate>)>, &'tcx ty::ResolverGlobalCtxt) {
     let arenas = Resolver::arenas();
     let _ = tcx.registered_tools(()); // Uses `crate_for_resolver`.
-    let (krate, pre_configured_attrs) = tcx.crate_for_resolver(()).steal();
+    let (krate, pre_configured_attrs) = tcx.crate_for_resolver.borrow_mut().take().unwrap();
     let mut resolver = Resolver::new(
         tcx,
         &pre_configured_attrs,
@@ -901,7 +901,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     rustc_mir_transform::provide(providers);
     rustc_monomorphize::provide(providers);
     rustc_privacy::provide(&mut providers.queries);
-    rustc_query_impl::provide(providers);
+    rustc_middle::query::impl_::provide(providers);
     rustc_resolve::provide(&mut providers.queries);
     rustc_hir_analysis::provide(&mut providers.queries);
     rustc_hir_typeck::provide(&mut providers.queries);
@@ -969,8 +969,6 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
         callback(sess, &mut providers);
     }
 
-    let incremental = dep_graph.is_fully_enabled();
-
     // Note: this function body is the origin point of the widely-used 'tcx lifetime.
     //
     // `gcx_cell` is defined here and `&gcx_cell` is passed to `create_global_ctxt`, which then
@@ -995,12 +993,11 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
         &hir_arena,
         untracked,
         dep_graph,
-        rustc_query_impl::make_dep_kind_vtables(&arena),
-        rustc_query_impl::query_system(
+        rustc_middle::query::impl_::make_dep_kind_vtables(&arena),
+        rustc_middle::query::impl_::query_system(
             providers.queries,
             providers.extern_queries,
             query_result_on_disk_cache,
-            incremental,
         ),
         providers.hooks,
         compiler.current_gcx.clone(),
@@ -1016,8 +1013,8 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
                 &pre_configured_attrs,
                 crate_name,
             )));
-            feed.crate_for_resolver(tcx.arena.alloc(Steal::new((krate, pre_configured_attrs))));
             feed.output_filenames(Arc::new(outputs));
+            *tcx.crate_for_resolver.borrow_mut() = Some((krate, pre_configured_attrs));
 
             let res = f(tcx);
             // FIXME maybe run finish even when a fatal error occurred? or at least