diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index 6318645f5..a90e62f8a 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -26,7 +26,7 @@ pub struct StoreFjallV3 { keyspace: Keyspace, puts: FxHashMap, dels: FxHashSet, - cache: Option>, + caches: Vec>, } impl StoreFjallV3 @@ -44,7 +44,7 @@ where mode: Mode3, kind: Kind3, ) -> Result { - Self::import_inner(db, path, name, version, mode, kind, None) + Self::import_inner(db, path, name, version, mode, kind, 0) } pub fn import_cached( @@ -54,9 +54,9 @@ where version: Version, mode: Mode3, kind: Kind3, - max_batches: u16, + max_batches: u8, ) -> Result { - Self::import_inner(db, path, name, version, mode, kind, Some(max_batches)) + Self::import_inner(db, path, name, version, mode, kind, max_batches) } fn import_inner( @@ -66,7 +66,7 @@ where version: Version, mode: Mode3, kind: Kind3, - max_batches: Option, + max_batches: u8, ) -> Result { fs::create_dir_all(path)?; @@ -82,7 +82,10 @@ where }, )?; - let cache = max_batches.map(|max_batches| Cache::new(max_batches)); + let mut caches = vec![]; + for _ in 0..max_batches { + caches.push(FxHashMap::default()); + } Ok(Self { meta, @@ -90,7 +93,7 @@ where keyspace, puts: FxHashMap::default(), dels: FxHashSet::default(), - cache, + caches, }) } @@ -139,10 +142,10 @@ where return Ok(Some(Cow::Borrowed(v))); } - if let Some(cache) = &self.cache - && let Some(v) = cache.get(key) - { - return Ok(Some(Cow::Borrowed(v))); + for cache in &self.caches { + if let Some(v) = cache.get(key) { + return Ok(Some(Cow::Borrowed(v))); + } } if let Some(slice) = self.keyspace.get(ByteView::from(key))? { @@ -200,7 +203,7 @@ where } #[inline] - fn needs(&self, height: Height) -> bool { + pub fn needs(&self, height: Height) -> bool { self.meta.needs(height) } @@ -210,13 +213,46 @@ where } Ok(()) } + + fn ingest<'a>( + keyspace: &Keyspace, + puts: impl Iterator, + dels: impl Iterator, + ) -> Result<()> + where + ByteView: From<&'a K> + From<&'a V>, + K: 'a, + V: 'a, + { + let mut items: Vec> = puts + .map(|(key, value)| Item::Value { key, value }) + .chain(dels.map(Item::Tomb)) + .collect(); + + items.sort_unstable(); + + let mut ingestion = keyspace.start_ingestion()?; + for item in items { + match item { + Item::Value { key, value } => { + ingestion.write(ByteView::from(key), ByteView::from(value))?; + } + Item::Tomb(key) => { + ingestion.write_tombstone(ByteView::from(key))?; + } + } + } + ingestion.finish()?; + + Ok(()) + } } impl AnyStore for StoreFjallV3 where K: Debug + Clone + From + Ord + Eq + Hash, V: Debug + Clone + From, - ByteView: From + From, + for<'a> ByteView: From + From + From<&'a K> + From<&'a V>, Self: Send + Sync, { fn keyspace(&self) -> &Keyspace { @@ -265,35 +301,13 @@ where return Ok(()); } - // Insert into cache here - if let Some(cache) = &mut self.cache { - for (k, v) in &puts { - cache.insert(k.clone(), v.clone()); - } - cache.commit(); + Self::ingest(&self.keyspace, puts.iter(), dels.iter())?; + + if !self.caches.is_empty() { + self.caches.pop(); + self.caches.insert(0, puts); } - let mut items: Vec<_> = puts - .into_iter() - .map(|(key, value)| Item::Value { key, value }) - .chain(dels.into_iter().map(Item::Tomb)) - .collect(); - - items.sort_unstable(); - - let mut ingestion = self.keyspace.start_ingestion()?; - for item in items { - match item { - Item::Value { key, value } => { - ingestion.write(ByteView::from(key), ByteView::from(value))?; - } - Item::Tomb(key) => { - ingestion.write_tombstone(ByteView::from(key))?; - } - } - } - ingestion.finish()?; - Ok(()) } } @@ -364,42 +378,3 @@ impl Kind3 { !matches!(*self, Self::Vec) } } - -#[derive(Clone)] -struct Cache { - index: FxHashMap, - current_batch: u16, - max_batches: u16, -} -impl Cache { - fn new(max_batches: u16) -> Self { - Self { - index: FxHashMap::default(), - current_batch: 0, - max_batches, - } - } - - #[inline] - fn get(&self, key: &K) -> Option<&V> { - self.index.get(key).map(|(v, _)| v) - } - - #[inline] - fn insert(&mut self, key: K, value: V) { - self.index.insert(key, (value, self.current_batch)); - } - - fn commit(&mut self) { - let max = self.max_batches; - let current = self.current_batch; - self.index - .retain(|_, (_, batch)| current.wrapping_sub(*batch) < max); - self.current_batch = self.current_batch.wrapping_add(1); - } - - fn clear(&mut self) { - self.index.clear(); - self.current_batch = 0; - } -} diff --git a/crates/brk_types/src/addresshash.rs b/crates/brk_types/src/addresshash.rs index 6ec3e74d7..d63250899 100644 --- a/crates/brk_types/src/addresshash.rs +++ b/crates/brk_types/src/addresshash.rs @@ -20,14 +20,12 @@ impl From for AddressHash { Self(u64::from_be_bytes((&*value).try_into().unwrap())) } } - impl From for ByteView { #[inline] fn from(value: AddressHash) -> Self { Self::from(&value) } } - impl From<&AddressHash> for ByteView { #[inline] fn from(value: &AddressHash) -> Self { diff --git a/crates/brk_types/src/height.rs b/crates/brk_types/src/height.rs index 08e5e5322..19898d723 100644 --- a/crates/brk_types/src/height.rs +++ b/crates/brk_types/src/height.rs @@ -227,15 +227,22 @@ impl TryFrom<&std::path::Path> for Height { } impl From for Height { - #[inline] + #[inline(always)] fn from(value: ByteView) -> Self { Self(u32::from_be_bytes((&*value).try_into().unwrap())) } } impl From for ByteView { - #[inline] + #[inline(always)] fn from(value: Height) -> Self { + ByteView::from(&value) + } +} + +impl From<&Height> for ByteView { + #[inline(always)] + fn from(value: &Height) -> Self { Self::new(&value.0.to_be_bytes()) } } diff --git a/crates/brk_types/src/txindex.rs b/crates/brk_types/src/txindex.rs index b6cbe5e5c..665aafeda 100644 --- a/crates/brk_types/src/txindex.rs +++ b/crates/brk_types/src/txindex.rs @@ -114,14 +114,20 @@ impl From for usize { } impl From for TxIndex { - #[inline] + #[inline(always)] fn from(value: ByteView) -> Self { Self(u32::from_be_bytes((&*value).try_into().unwrap())) } } impl From for ByteView { - #[inline] + #[inline(always)] fn from(value: TxIndex) -> Self { + ByteView::from(&value) + } +} +impl From<&TxIndex> for ByteView { + #[inline(always)] + fn from(value: &TxIndex) -> Self { Self::new(&value.to_be_bytes()) } } diff --git a/crates/brk_types/src/typeindex.rs b/crates/brk_types/src/typeindex.rs index 1264e0529..10a56bf4e 100644 --- a/crates/brk_types/src/typeindex.rs +++ b/crates/brk_types/src/typeindex.rs @@ -117,8 +117,14 @@ impl From for TypeIndex { } } impl From for ByteView { - #[inline] + #[inline(always)] fn from(value: TypeIndex) -> Self { + ByteView::from(&value) + } +} +impl From<&TypeIndex> for ByteView { + #[inline(always)] + fn from(value: &TypeIndex) -> Self { Self::new(&value.0.to_be_bytes()) } } diff --git a/crates/brk_types/src/unit.rs b/crates/brk_types/src/unit.rs index 5bc52d041..0a8bdb077 100644 --- a/crates/brk_types/src/unit.rs +++ b/crates/brk_types/src/unit.rs @@ -4,14 +4,20 @@ use byteview::ByteView; pub struct Unit; impl From for Unit { - #[inline] + #[inline(always)] fn from(_: ByteView) -> Self { Self } } impl From for ByteView { - #[inline] + #[inline(always)] fn from(_: Unit) -> Self { Self::new(&[]) } } +impl From<&Unit> for ByteView { + #[inline(always)] + fn from(_: &Unit) -> Self { + Self::new(&[]) + } +}