Skip to content

Commit d44f638

Browse files
ibraheemdevibraheemdev
authored
Replace ingredient cache with faster ingredient map (#921)
* replace ingredient cache with faster ingredient map * avoid checking for downcasters in ingredient cache slow-path * pre-size ingredient map * avoid double lookup in ingredient creation slow-path
1 parent 0666e20 commit d44f638

12 files changed

+198
-41
lines changed

‎Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ hashbrown = "0.15"
1919
hashlink = "0.10"
2020
indexmap = "2"
2121
intrusive-collections = "0.9.7"
22+
papaya = "0.2.2"
2223
parking_lot = "0.12"
2324
portable-atomic = "1"
2425
rustc-hash = "2"

‎components/salsa-macro-rules/src/setup_accumulator_impl.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ macro_rules! setup_accumulator_impl {
2626
$zalsa::IngredientCache::new();
2727

2828
$CACHE.get_or_create(zalsa, || {
29-
zalsa.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
29+
zalsa
30+
.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
31+
.get_or_create()
3032
})
3133
}
3234

‎components/salsa-macro-rules/src/setup_input_struct.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,14 +101,14 @@ macro_rules! setup_input_struct {
101101
$zalsa::IngredientCache::new();
102102

103103
CACHE.get_or_create(zalsa, || {
104-
zalsa.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
104+
zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
105105
})
106106
}
107107

108108
pub fn ingredient_mut(db: &mut dyn $zalsa::Database) -> (&mut $zalsa_struct::IngredientImpl<Self>, &mut $zalsa::Runtime) {
109109
let zalsa_mut = db.zalsa_mut();
110110
zalsa_mut.new_revision();
111-
let index = zalsa_mut.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>();
111+
let index = zalsa_mut.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create();
112112
let (ingredient, runtime) = zalsa_mut.lookup_ingredient_mut(index);
113113
let ingredient = ingredient.assert_type_mut::<$zalsa_struct::IngredientImpl<Self>>();
114114
(ingredient, runtime)
@@ -150,7 +150,7 @@ macro_rules! setup_input_struct {
150150
type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
151151

152152
fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
153-
aux.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
153+
aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
154154
}
155155

156156
#[inline]

‎components/salsa-macro-rules/src/setup_interned_struct.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ macro_rules! setup_interned_struct {
149149

150150
let zalsa = db.zalsa();
151151
CACHE.get_or_create(zalsa, || {
152-
zalsa.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
152+
zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
153153
})
154154
}
155155
}
@@ -182,7 +182,7 @@ macro_rules! setup_interned_struct {
182182
type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
183183

184184
fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
185-
aux.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
185+
aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
186186
}
187187

188188
#[inline]

‎components/salsa-macro-rules/src/setup_tracked_fn.rs

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,15 +154,24 @@ macro_rules! setup_tracked_fn {
154154
fn fn_ingredient(db: &dyn $Db) -> &$zalsa::function::IngredientImpl<$Configuration> {
155155
let zalsa = db.zalsa();
156156
$FN_CACHE.get_or_create(zalsa, || {
157+
let jar_entry = zalsa.lookup_jar_by_type::<$Configuration>();
158+
159+
// If the ingredient has already been inserted, we know that the downcaster
160+
// has also been registered. This is a fast-path for multi-database use cases
161+
// that bypass the ingredient cache and will always execute this closure.
162+
if let Some(index) = jar_entry.get() {
163+
return index;
164+
}
165+
157166
<dyn $Db as $Db>::zalsa_register_downcaster(db);
158-
zalsa.add_or_lookup_jar_by_type::<$Configuration>()
167+
jar_entry.get_or_create()
159168
})
160169
}
161170

162171
pub fn fn_ingredient_mut(db: &mut dyn $Db) -> &mut $zalsa::function::IngredientImpl<Self> {
163172
<dyn $Db as $Db>::zalsa_register_downcaster(db);
164173
let zalsa_mut = db.zalsa_mut();
165-
let index = zalsa_mut.add_or_lookup_jar_by_type::<$Configuration>();
174+
let index = zalsa_mut.lookup_jar_by_type::<$Configuration>().get_or_create();
166175
let (ingredient, _) = zalsa_mut.lookup_ingredient_mut(index);
167176
ingredient.assert_type_mut::<$zalsa::function::IngredientImpl<Self>>()
168177
}
@@ -174,7 +183,7 @@ macro_rules! setup_tracked_fn {
174183
let zalsa = db.zalsa();
175184
$INTERN_CACHE.get_or_create(zalsa, || {
176185
<dyn $Db as $Db>::zalsa_register_downcaster(db);
177-
zalsa.add_or_lookup_jar_by_type::<$Configuration>().successor(0)
186+
zalsa.lookup_jar_by_type::<$Configuration>().get_or_create().successor(0)
178187
})
179188
}
180189
}

‎components/salsa-macro-rules/src/setup_tracked_struct.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ macro_rules! setup_tracked_struct {
188188
$zalsa::IngredientCache::new();
189189

190190
CACHE.get_or_create(zalsa, || {
191-
zalsa.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
191+
zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
192192
})
193193
}
194194
}
@@ -211,7 +211,7 @@ macro_rules! setup_tracked_struct {
211211
type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
212212

213213
fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
214-
aux.add_or_lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
214+
aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
215215
}
216216

217217
#[inline]

‎src/accumulator.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ pub struct IngredientImpl<A: Accumulator> {
6464
impl<A: Accumulator> IngredientImpl<A> {
6565
/// Find the accumulator ingredient for `A` in the database, if any.
6666
pub fn from_zalsa(zalsa: &Zalsa) -> Option<&Self> {
67-
let index = zalsa.add_or_lookup_jar_by_type::<JarImpl<A>>();
67+
let index = zalsa.lookup_jar_by_type::<JarImpl<A>>().get_or_create();
6868
let ingredient = zalsa.lookup_ingredient(index).assert_type::<Self>();
6969
Some(ingredient)
7070
}

‎src/hash.rs

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::hash::{BuildHasher, Hash};
1+
use std::hash::{BuildHasher, Hash, Hasher};
22

33
pub(crate) type FxHasher = std::hash::BuildHasherDefault<rustc_hash::FxHasher>;
44
pub(crate) type FxIndexSet<K> = indexmap::IndexSet<K, FxHasher>;
@@ -8,3 +8,24 @@ pub(crate) type FxHashSet<K> = std::collections::HashSet<K, FxHasher>;
88
pub(crate) fn hash<T: Hash>(t: &T) -> u64 {
99
FxHasher::default().hash_one(t)
1010
}
11+
12+
// `TypeId` is a 128-bit hash internally, and it's `Hash` implementation
13+
// writes the lower 64-bits. Hashing it again would be unnecessary.
14+
#[derive(Default)]
15+
pub(crate) struct TypeIdHasher(u64);
16+
17+
impl Hasher for TypeIdHasher {
18+
fn write(&mut self, _: &[u8]) {
19+
unreachable!("`TypeId` calls `write_u64`");
20+
}
21+
22+
#[inline]
23+
fn write_u64(&mut self, id: u64) {
24+
self.0 = id;
25+
}
26+
27+
#[inline]
28+
fn finish(&self) -> u64 {
29+
self.0
30+
}
31+
}

‎src/salsa_struct.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ pub trait SalsaStructInDb: Sized {
1010
/// Lookup or create ingredient indices.
1111
///
1212
/// Note that this method does *not* create the ingredients themselves, this is handled by
13-
/// [`Zalsa::add_or_lookup_jar_by_type()`]. This method only creates
13+
/// [`crate::zalsa::JarEntry::get_or_create`]. This method only creates
1414
/// or looks up the indices corresponding to the ingredients.
1515
///
16-
/// While implementors of this trait may call [`Zalsa::add_or_lookup_jar_by_type()`]
16+
/// While implementors of this trait may call [`crate::zalsa::JarEntry::get_or_create`]
1717
/// to create the ingredient, they aren't required to. For example, supertypes recursively
18-
/// call [`Zalsa::add_or_lookup_jar_by_type()`] for their variants and combine them.
18+
/// call [`crate::zalsa::JarEntry::get_or_create`] for their variants and combine them.
1919
fn lookup_or_create_ingredient_index(zalsa: &Zalsa) -> IngredientIndices;
2020

2121
/// Plumbing to support nested salsa supertypes.

‎src/sync.rs

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,40 @@ pub mod shim {
55
pub use shuttle::sync::*;
66
pub use shuttle::{thread, thread_local};
77

8+
pub mod papaya {
9+
use std::hash::{BuildHasher, Hash};
10+
use std::marker::PhantomData;
11+
12+
pub struct HashMap<K, V, S>(super::Mutex<std::collections::HashMap<K, V, S>>);
13+
14+
impl<K, V, S: Default> Default for HashMap<K, V, S> {
15+
fn default() -> Self {
16+
Self(super::Mutex::default())
17+
}
18+
}
19+
20+
pub struct LocalGuard<'a>(PhantomData<&'a ()>);
21+
22+
impl<K, V, S> HashMap<K, V, S>
23+
where
24+
K: Eq + Hash,
25+
V: Clone,
26+
S: BuildHasher,
27+
{
28+
pub fn guard(&self) -> LocalGuard<'_> {
29+
LocalGuard(PhantomData)
30+
}
31+
32+
pub fn get(&self, key: &K, _guard: &LocalGuard<'_>) -> Option<V> {
33+
self.0.lock().get(key).cloned()
34+
}
35+
36+
pub fn insert(&self, key: K, value: V, _guard: &LocalGuard<'_>) {
37+
self.0.lock().insert(key, value);
38+
}
39+
}
40+
}
41+
842
/// A wrapper around shuttle's `Mutex` to mirror parking-lot's API.
943
#[derive(Default, Debug)]
1044
pub struct Mutex<T>(shuttle::sync::Mutex<T>);
@@ -139,6 +173,48 @@ pub mod shim {
139173
pub use std::sync::atomic::*;
140174
}
141175

176+
pub mod papaya {
177+
use std::hash::{BuildHasher, Hash};
178+
179+
pub use papaya::LocalGuard;
180+
181+
pub struct HashMap<K, V, S>(papaya::HashMap<K, V, S>);
182+
183+
impl<K, V, S: Default> Default for HashMap<K, V, S> {
184+
fn default() -> Self {
185+
Self(
186+
papaya::HashMap::builder()
187+
.capacity(256) // A relatively large capacity to hopefully avoid resizing.
188+
.resize_mode(papaya::ResizeMode::Blocking)
189+
.hasher(S::default())
190+
.build(),
191+
)
192+
}
193+
}
194+
195+
impl<K, V, S> HashMap<K, V, S>
196+
where
197+
K: Eq + Hash,
198+
V: Clone,
199+
S: BuildHasher,
200+
{
201+
#[inline]
202+
pub fn guard(&self) -> LocalGuard<'_> {
203+
self.0.guard()
204+
}
205+
206+
#[inline]
207+
pub fn get(&self, key: &K, guard: &LocalGuard<'_>) -> Option<V> {
208+
self.0.get(key, guard).cloned()
209+
}
210+
211+
#[inline]
212+
pub fn insert(&self, key: K, value: V, guard: &LocalGuard<'_>) {
213+
self.0.insert(key, value, guard);
214+
}
215+
}
216+
}
217+
142218
/// A wrapper around parking-lot's `Condvar` to mirror shuttle's API.
143219
pub struct Condvar(parking_lot::Condvar);
144220

0 commit comments

Comments
 (0)