diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 0860520ef9dfe..b8795c1d6dbe8 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -21,6 +21,7 @@ rustc_queries! { /// To avoid this fate, do not call `tcx.hir().krate()`; instead, /// prefer wrappers like `tcx.visit_all_items_in_krate()`. query hir_crate(key: ()) -> &'tcx Crate<'tcx> { + storage(SingletonCacheSelector) eval_always no_hash desc { "get the crate HIR" } @@ -29,6 +30,7 @@ rustc_queries! { /// The indexed HIR. This can be conveniently accessed by `tcx.hir()`. /// Avoid calling this query directly. query index_hir(_: ()) -> &'tcx crate::hir::IndexedHir<'tcx> { + storage(SingletonCacheSelector) eval_always no_hash desc { "index HIR" } @@ -115,6 +117,7 @@ rustc_queries! { } query analysis(key: ()) -> Result<(), ErrorReported> { + storage(SingletonCacheSelector) eval_always desc { "running analysis passes on this crate" } } @@ -701,6 +704,7 @@ rustc_queries! { } query typeck_item_bodies(_: ()) -> () { + storage(SingletonCacheSelector) desc { "type-checking all item bodies" } } @@ -769,6 +773,7 @@ rustc_queries! { /// Not meant to be used directly outside of coherence. query crate_inherent_impls_overlap_check(_: ()) -> () { + storage(SingletonCacheSelector) eval_always desc { "check for overlap between inherent impls defined in this crate" } } @@ -862,10 +867,12 @@ rustc_queries! { /// Performs part of the privacy check and computes "access levels". query privacy_access_levels(_: ()) -> &'tcx AccessLevels { + storage(SingletonCacheSelector) eval_always desc { "privacy access levels" } } query check_private_in_public(_: ()) -> () { + storage(SingletonCacheSelector) eval_always desc { "checking for private elements in public interfaces" } } @@ -981,6 +988,7 @@ rustc_queries! { /// /// [`LOCAL_CRATE`]: rustc_hir::def_id::LOCAL_CRATE query all_local_trait_impls(_: ()) -> &'tcx BTreeMap> { + storage(SingletonCacheSelector) desc { "local trait impls" } } @@ -1084,6 +1092,7 @@ rustc_queries! { } query dependency_formats(_: ()) -> Lrc { + storage(SingletonCacheSelector) desc { "get the linkage format of all dependencies" } } @@ -1229,12 +1238,15 @@ rustc_queries! { /// Identifies the entry-point (e.g., the `main` function) for a given /// crate, returning `None` if there is no entry point (such as for library crates). query entry_fn(_: ()) -> Option<(DefId, EntryFnType)> { + storage(SingletonCacheSelector) desc { "looking up the entry function of a crate" } } query plugin_registrar_fn(_: ()) -> Option { + storage(SingletonCacheSelector) desc { "looking up the plugin registrar for a crate" } } query proc_macro_decls_static(_: ()) -> Option { + storage(SingletonCacheSelector) desc { "looking up the derive registrar for a crate" } } query crate_disambiguator(_: CrateNum) -> CrateDisambiguator { @@ -1411,6 +1423,7 @@ rustc_queries! { desc { "looking at the source for a crate" } } query postorder_cnums(_: ()) -> &'tcx [CrateNum] { + storage(SingletonCacheSelector) eval_always desc { "generating a postorder list of CrateNums" } } @@ -1430,6 +1443,7 @@ rustc_queries! { desc { |tcx| "maybe_unused_trait_import for `{}`", tcx.def_path_str(def_id.to_def_id()) } } query maybe_unused_extern_crates(_: ()) -> &'tcx [(LocalDefId, Span)] { + storage(SingletonCacheSelector) eval_always desc { "looking up all possibly unused extern crates" } } @@ -1445,6 +1459,7 @@ rustc_queries! { desc { "calculating the stability index for the local crate" } } query all_crate_nums(_: ()) -> &'tcx [CrateNum] { + storage(SingletonCacheSelector) eval_always desc { "fetching all foreign CrateNum instances" } } @@ -1453,6 +1468,7 @@ rustc_queries! { /// (i.e., including those from subcrates). This is used only for /// error reporting. query all_traits(_: ()) -> &'tcx [DefId] { + storage(SingletonCacheSelector) desc { "fetching all foreign and local traits" } } @@ -1467,6 +1483,7 @@ rustc_queries! { } query collect_and_partition_mono_items(_: ()) -> (&'tcx DefIdSet, &'tcx [CodegenUnit<'tcx>]) { + storage(SingletonCacheSelector) eval_always desc { "collect_and_partition_mono_items" } } @@ -1476,8 +1493,9 @@ rustc_queries! { /// All items participating in code generation together with items inlined into them. query codegened_and_inlined_items(_: ()) -> &'tcx DefIdSet { + storage(SingletonCacheSelector) eval_always - desc { "codegened_and_inlined_items" } + desc { "codegened_and_inlined_items" } } query codegen_unit(_: Symbol) -> &'tcx CodegenUnit<'tcx> { @@ -1491,10 +1509,12 @@ rustc_queries! { } } query backend_optimization_level(_: ()) -> OptLevel { + storage(SingletonCacheSelector) desc { "optimization level used by backend" } } query output_filenames(_: ()) -> Arc { + storage(SingletonCacheSelector) eval_always desc { "output_filenames" } } @@ -1671,6 +1691,7 @@ rustc_queries! { } query features_query(_: ()) -> &'tcx rustc_feature::Features { + storage(SingletonCacheSelector) eval_always desc { "looking up enabled feature gates" } } diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs index 297110ee3ecff..2301c890697d8 100644 --- a/compiler/rustc_middle/src/ty/query/mod.rs +++ b/compiler/rustc_middle/src/ty/query/mod.rs @@ -116,7 +116,7 @@ macro_rules! query_storage { ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => { <$ty as CacheSelector<$K, $V>>::Cache }; - ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)?][$($args:tt)*]) => { query_storage!([$($($modifiers)*)*][$($args)*]) }; } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index b4191c135b4f9..20faf6356044d 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -228,7 +228,7 @@ macro_rules! handle_cycle_error { $error.delay_as_bug(); Value::from_cycle_error($tcx) }}; - ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)?][$($args:tt)*]) => { handle_cycle_error!([$($($modifiers)*)*][$($args)*]) }; } @@ -240,7 +240,7 @@ macro_rules! is_anon { ([anon $($rest:tt)*]) => {{ true }}; - ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => { + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)?]) => { is_anon!([$($($modifiers)*)*]) }; } @@ -252,7 +252,7 @@ macro_rules! is_eval_always { ([eval_always $($rest:tt)*]) => {{ true }}; - ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => { + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)?]) => { is_eval_always!([$($($modifiers)*)*]) }; } @@ -264,7 +264,7 @@ macro_rules! hash_result { ([no_hash $($rest:tt)*][$hcx:expr, $result:expr]) => {{ None }}; - ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)?][$($args:tt)*]) => { hash_result!([$($($modifiers)*)*][$($args)*]) }; } diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs index 0d4fb34265c55..27632fb9de20e 100644 --- a/compiler/rustc_query_system/src/lib.rs +++ b/compiler/rustc_query_system/src/lib.rs @@ -1,4 +1,5 @@ #![feature(bool_to_option)] +#![feature(const_panic)] #![feature(core_intrinsics)] #![feature(hash_raw_entry)] #![feature(iter_zip)] diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 011c2ceebb714..84104747f3913 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -221,3 +221,88 @@ where } } } + +pub struct SingletonCacheSelector; + +impl CacheSelector for SingletonCacheSelector { + type Cache = SingletonCache; +} + +pub struct SingletonCache(PhantomData<(K, V)>); + +impl Default for SingletonCache { + fn default() -> Self { + SingletonCache(PhantomData) + } +} + +impl SingletonCache { + const KEY_IS_ZST: () = { + if std::mem::size_of::() != 0 { + panic!("Key must be a ZST"); + } + }; +} + +impl QueryStorage for SingletonCache { + type Value = V; + type Stored = V; + + #[inline] + fn store_nocache(&self, value: Self::Value) -> Self::Stored { + // We have no dedicated storage + value + } +} + +impl QueryCache for SingletonCache +where + K: Eq + Hash + Copy + Clone + Debug, + V: Clone + Debug, +{ + type Key = K; + type Sharded = Option<(K, V, DepNodeIndex)>; + + #[inline(always)] + fn lookup<'s, R, OnHit>( + &self, + state: &'s QueryCacheStore, + _key: &K, + on_hit: OnHit, + ) -> Result + where + OnHit: FnOnce(&V, DepNodeIndex) -> R, + { + let _ = Self::KEY_IS_ZST; + + let lock = state.shards.get_shard_by_index(0).lock(); + if let Some((_key, val, dep_node)) = &*lock { + Ok(on_hit(&val, *dep_node)) + } else { + Err(QueryLookup { key_hash: 0, shard: 0 }) + } + } + + #[inline] + fn complete( + &self, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ) -> Self::Stored { + *lock_sharded_storage = Some((key, value.clone(), index)); + value + } + + fn iter( + &self, + shards: &Sharded, + f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex), + ) { + let lock = shards.get_shard_by_index(0).lock(); + if let Some((key, val, dep_node)) = &*lock { + f(&key, &val, *dep_node); + } + } +} diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index 927e8117f05ea..8542f09ce9eeb 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -9,6 +9,7 @@ pub use self::job::{print_query_stack, QueryInfo, QueryJob, QueryJobId, QueryJob mod caches; pub use self::caches::{ ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, + SingletonCacheSelector, }; mod config; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index c1f9fa39e98c5..b62011c91bea3 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -31,7 +31,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub struct QueryCacheStore { cache: C, - shards: Sharded, + pub(super) shards: Sharded, #[cfg(debug_assertions)] pub cache_hits: AtomicUsize, } @@ -50,7 +50,7 @@ impl Default for QueryCacheStore { /// Values used when checking a query cache which can be reused on a cache-miss to execute the query. pub struct QueryLookup { pub(super) key_hash: u64, - shard: usize, + pub(super) shard: usize, } // We compute the key's hash once and then use it for both the