mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-13 12:08:13 -07:00
global: big vec refactor + lazy
This commit is contained in:
nym21
+1
-218
@@ -8,225 +8,8 @@ mod structs;
|
||||
mod traits;
|
||||
mod variants;
|
||||
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
pub use enums::*;
|
||||
pub use memmap2::Mmap;
|
||||
pub use structs::*;
|
||||
pub use traits::*;
|
||||
use variants::*;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum StoredVec<I, T> {
|
||||
Raw(RawVec<I, T>),
|
||||
Compressed(CompressedVec<I, T>),
|
||||
}
|
||||
|
||||
impl<I, T> StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
|
||||
if *compressed {
|
||||
Ok(Self::Compressed(CompressedVec::forced_import(
|
||||
path, version,
|
||||
)?))
|
||||
} else {
|
||||
Ok(Self::Raw(RawVec::forced_import(path, version)?))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> StoredVecIterator<'_, I, T> {
|
||||
self.into_iter()
|
||||
}
|
||||
|
||||
pub fn iter_at(&self, i: I) -> StoredVecIterator<'_, I, T> {
|
||||
let mut iter = self.into_iter();
|
||||
iter.set(i);
|
||||
iter
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> DynamicVec for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type I = I;
|
||||
type T = T;
|
||||
|
||||
#[inline]
|
||||
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.read_(index, guard),
|
||||
StoredVec::Compressed(v) => v.read_(index, guard),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mmap(&self) -> &ArcSwap<Mmap> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.mmap(),
|
||||
StoredVec::Compressed(v) => v.mmap(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn stored_len(&self) -> usize {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.stored_len(),
|
||||
StoredVec::Compressed(v) => v.stored_len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn pushed(&self) -> &[T] {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.pushed(),
|
||||
StoredVec::Compressed(v) => v.pushed(),
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn mut_pushed(&mut self) -> &mut Vec<T> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.mut_pushed(),
|
||||
StoredVec::Compressed(v) => v.mut_pushed(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path(&self) -> &Path {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.path(),
|
||||
StoredVec::Compressed(v) => v.path(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> GenericVec<I, T> for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.collect_range(from, to),
|
||||
StoredVec::Compressed(v) => v.collect_range(from, to),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.flush(),
|
||||
StoredVec::Compressed(v) => v.flush(),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.truncate_if_needed(index),
|
||||
StoredVec::Compressed(v) => v.truncate_if_needed(index),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.version(),
|
||||
StoredVec::Compressed(v) => v.version(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
DynamicVec::len(self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
GenericVec::index_type_to_string(self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
GenericVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
GenericVec::modified_time(self)
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
GenericVec::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum StoredVecIterator<'a, I, T> {
|
||||
Raw(RawVecIterator<'a, I, T>),
|
||||
Compressed(CompressedVecIterator<'a, I, T>),
|
||||
}
|
||||
|
||||
impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self {
|
||||
Self::Compressed(i) => i.next(),
|
||||
Self::Raw(i) => i.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> BaseVecIterator for StoredVecIterator<'_, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
match self {
|
||||
Self::Compressed(iter) => iter.mut_index(),
|
||||
Self::Raw(iter) => iter.mut_index(),
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Compressed(i) => i.len(),
|
||||
Self::Raw(i) => i.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T> IntoIterator for &'a StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = StoredVecIterator<'a, I, T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
match self {
|
||||
StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()),
|
||||
StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub use variants::*;
|
||||
|
||||
@@ -1,24 +1,73 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::Result;
|
||||
use crate::{Result, Version};
|
||||
|
||||
use super::{BoxedVecIterator, StoredIndex, StoredType};
|
||||
|
||||
pub trait AnyVec: Send + Sync {
|
||||
fn version(&self) -> Version;
|
||||
fn name(&self) -> String;
|
||||
fn index_type_to_string(&self) -> &str;
|
||||
fn len(&self) -> usize;
|
||||
fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>>;
|
||||
fn modified_time(&self) -> Result<Duration>;
|
||||
fn any_vec(&self) -> &dyn AnyVec
|
||||
fn index_type_to_string(&self) -> &str;
|
||||
}
|
||||
|
||||
pub trait AnyIterableVec<I, T>: AnyVec {
|
||||
#[allow(clippy::wrong_self_convention)]
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
Self: Sized,
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a;
|
||||
|
||||
fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
{
|
||||
self
|
||||
self.boxed_iter()
|
||||
}
|
||||
|
||||
fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
{
|
||||
let mut iter = self.boxed_iter();
|
||||
iter.set(i);
|
||||
iter
|
||||
}
|
||||
|
||||
fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
{
|
||||
let mut iter = self.boxed_iter();
|
||||
iter.set_(i);
|
||||
iter
|
||||
}
|
||||
}
|
||||
|
||||
pub trait CloneableAnyIterableVec<I, T>: AnyIterableVec<I, T> {
|
||||
fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>>;
|
||||
}
|
||||
|
||||
impl<I, T, U> CloneableAnyIterableVec<I, T> for U
|
||||
where
|
||||
U: 'static + AnyIterableVec<I, T> + Clone,
|
||||
{
|
||||
fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> Clone for Box<dyn CloneableAnyIterableVec<I, T>> {
|
||||
fn clone(&self) -> Self {
|
||||
self.boxed_clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub type BoxedAnyIterableVec<I, T> = Box<dyn CloneableAnyIterableVec<I, T>>;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
use crate::{Error, Result};
|
||||
|
||||
use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType};
|
||||
|
||||
pub trait CollectableVec<I, T>: AnyVec + AnyIterableVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
|
||||
let len = self.len();
|
||||
let from = from.unwrap_or_default();
|
||||
let to = to.map_or(len, |i| i.min(len));
|
||||
|
||||
if from >= len || from >= to {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.iter_at_(from)
|
||||
.take(to - from)
|
||||
.map(|(_, v)| v.into_inner())
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn i64_to_usize(i: i64, len: usize) -> usize {
|
||||
if i >= 0 {
|
||||
i as usize
|
||||
} else {
|
||||
let v = len as i64 + i;
|
||||
if v < 0 { 0 } else { v as usize }
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
|
||||
let len = self.len();
|
||||
let from = from.map(|i| Self::i64_to_usize(i, len));
|
||||
let to = to.map(|i| Self::i64_to_usize(i, len));
|
||||
self.collect_range(from, to)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
self.collect_signed_range(from, to)?
|
||||
.into_iter()
|
||||
.map(|v| serde_json::to_value(v).map_err(Error::from))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, V> CollectableVec<I, T> for V
|
||||
where
|
||||
V: AnyVec + AnyIterableVec<I, T>,
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
}
|
||||
|
||||
pub trait AnyCollectableVec: AnyVec {
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>>;
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::{Result, Value};
|
||||
|
||||
use super::{StoredIndex, StoredType};
|
||||
|
||||
pub trait DynamicVec: Send + Sync {
|
||||
type I: StoredIndex;
|
||||
type T: StoredType;
|
||||
const SIZE_OF_T: usize = size_of::<Self::T>();
|
||||
|
||||
#[inline]
|
||||
fn read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Self::T>> {
|
||||
self.read_(index.to_usize()?, mmap)
|
||||
}
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
|
||||
|
||||
#[inline]
|
||||
fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
|
||||
self.get_or_read_(index.to_usize()?, mmap)
|
||||
}
|
||||
#[inline]
|
||||
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
|
||||
let stored_len = mmap.len() / Self::SIZE_OF_T;
|
||||
|
||||
if index >= stored_len {
|
||||
let pushed = self.pushed();
|
||||
let j = index - stored_len;
|
||||
if j >= pushed.len() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(pushed.get(j).map(Value::Ref))
|
||||
} else {
|
||||
Ok(self.read_(index, mmap)?.map(Value::Owned))
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.stored_len() + self.pushed_len()
|
||||
}
|
||||
#[inline]
|
||||
fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
fn mmap(&self) -> &ArcSwap<Mmap>;
|
||||
|
||||
fn stored_len(&self) -> usize;
|
||||
|
||||
fn pushed(&self) -> &[Self::T];
|
||||
#[inline]
|
||||
fn pushed_len(&self) -> usize {
|
||||
self.pushed().len()
|
||||
}
|
||||
fn mut_pushed(&mut self) -> &mut Vec<Self::T>;
|
||||
#[inline]
|
||||
fn push(&mut self, value: Self::T) {
|
||||
self.mut_pushed().push(value)
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path;
|
||||
}
|
||||
@@ -6,17 +6,70 @@ use std::{
|
||||
time::{self, Duration},
|
||||
};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::{Error, Result, Version};
|
||||
use crate::{Result, Value};
|
||||
|
||||
use super::{DynamicVec, StoredIndex, StoredType};
|
||||
use super::{StoredIndex, StoredType};
|
||||
|
||||
pub trait GenericVec<I, T>: DynamicVec<I = I, T = T>
|
||||
pub trait GenericStoredVec<I, T>: Send + Sync
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
const SIZE_OF_T: usize = size_of::<T>();
|
||||
|
||||
#[inline]
|
||||
fn read(&self, index: I, mmap: &Mmap) -> Result<Option<T>> {
|
||||
self.read_(index.to_usize()?, mmap)
|
||||
}
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>>;
|
||||
|
||||
#[inline]
|
||||
fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
|
||||
self.get_or_read_(index.to_usize()?, mmap)
|
||||
}
|
||||
#[inline]
|
||||
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
|
||||
let stored_len = mmap.len() / Self::SIZE_OF_T;
|
||||
|
||||
if index >= stored_len {
|
||||
let pushed = self.pushed();
|
||||
let j = index - stored_len;
|
||||
if j >= pushed.len() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(pushed.get(j).map(Value::Ref))
|
||||
} else {
|
||||
Ok(self.read_(index, mmap)?.map(Value::Owned))
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len_(&self) -> usize {
|
||||
self.stored_len() + self.pushed_len()
|
||||
}
|
||||
|
||||
fn mmap(&self) -> &ArcSwap<Mmap>;
|
||||
|
||||
fn stored_len(&self) -> usize;
|
||||
|
||||
fn pushed(&self) -> &[T];
|
||||
#[inline]
|
||||
fn pushed_len(&self) -> usize {
|
||||
self.pushed().len()
|
||||
}
|
||||
fn mut_pushed(&mut self) -> &mut Vec<T>;
|
||||
#[inline]
|
||||
fn push(&mut self, value: T) {
|
||||
self.mut_pushed().push(value)
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path;
|
||||
|
||||
// ---
|
||||
|
||||
fn open_file(&self) -> io::Result<File> {
|
||||
Self::open_file_(&self.path_vec())
|
||||
}
|
||||
@@ -74,54 +127,17 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn has(&self, index: Self::I) -> Result<bool> {
|
||||
fn has(&self, index: I) -> Result<bool> {
|
||||
Ok(self.has_(index.to_usize()?))
|
||||
}
|
||||
#[inline]
|
||||
fn has_(&self, index: usize) -> bool {
|
||||
index < self.len()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
Self::I::to_string()
|
||||
index < self.len_()
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()>;
|
||||
|
||||
fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>;
|
||||
|
||||
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>>;
|
||||
|
||||
#[inline]
|
||||
fn i64_to_usize(i: i64, len: usize) -> usize {
|
||||
if i >= 0 {
|
||||
i as usize
|
||||
} else {
|
||||
let v = len as i64 + i;
|
||||
if v < 0 { 0 } else { v as usize }
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<Self::T>> {
|
||||
let len = self.len();
|
||||
let from = from.map(|i| Self::i64_to_usize(i, len));
|
||||
let to = to.map(|i| Self::i64_to_usize(i, len));
|
||||
self.collect_range(from, to)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
self.collect_signed_range(from, to)?
|
||||
.into_iter()
|
||||
.map(|v| serde_json::to_value(v).map_err(Error::from))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
}
|
||||
fn truncate_if_needed(&mut self, index: I) -> Result<()>;
|
||||
|
||||
#[inline]
|
||||
fn path_vec(&self) -> PathBuf {
|
||||
@@ -143,7 +159,7 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn name(&self) -> String {
|
||||
fn name_(&self) -> String {
|
||||
self.path()
|
||||
.file_name()
|
||||
.unwrap()
|
||||
@@ -152,13 +168,11 @@ where
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
fn modified_time_(&self) -> Result<Duration> {
|
||||
Ok(self
|
||||
.path_vec()
|
||||
.metadata()?
|
||||
.modified()?
|
||||
.duration_since(time::UNIX_EPOCH)?)
|
||||
}
|
||||
|
||||
fn version(&self) -> Version;
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ where
|
||||
fn to_string<'a>() -> &'a str;
|
||||
fn decremented(self) -> Option<Self>;
|
||||
}
|
||||
|
||||
impl<I> StoredIndex for I
|
||||
where
|
||||
I: Debug
|
||||
@@ -32,9 +32,9 @@ pub trait BaseVecIterator: Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> + 'a {
|
||||
pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> {
|
||||
type I: StoredIndex;
|
||||
type T: StoredType;
|
||||
type T: StoredType + 'a;
|
||||
|
||||
#[inline]
|
||||
fn set(&mut self, i: Self::I) {
|
||||
@@ -77,10 +77,13 @@ pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)>
|
||||
|
||||
impl<'a, I, T, Iter> VecIterator<'a> for Iter
|
||||
where
|
||||
Iter: BaseVecIterator<Item = (I, Value<'a, T>)> + 'a,
|
||||
Iter: BaseVecIterator<Item = (I, Value<'a, T>)>,
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
{
|
||||
type I = I;
|
||||
type T = T;
|
||||
}
|
||||
|
||||
pub type BoxedVecIterator<'a, I, T> =
|
||||
Box<dyn VecIterator<'a, I = I, T = T, Item = (I, Value<'a, T>)> + 'a>;
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
mod any;
|
||||
mod dynamic;
|
||||
mod collectable;
|
||||
mod generic;
|
||||
mod index;
|
||||
mod iterator;
|
||||
mod stored_index;
|
||||
mod stored_type;
|
||||
mod r#type;
|
||||
|
||||
pub use any::*;
|
||||
pub use dynamic::*;
|
||||
pub use collectable::*;
|
||||
pub use generic::*;
|
||||
pub use index::*;
|
||||
pub use iterator::*;
|
||||
pub use stored_index::*;
|
||||
pub use stored_type::*;
|
||||
pub use r#type::*;
|
||||
|
||||
@@ -17,6 +17,7 @@ where
|
||||
+ Serialize,
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> StoredType for T where
|
||||
T: Sized
|
||||
+ Debug
|
||||
@@ -3,6 +3,7 @@ use std::{
|
||||
mem,
|
||||
path::Path,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use arc_swap::{ArcSwap, Guard};
|
||||
@@ -11,8 +12,9 @@ use rayon::prelude::*;
|
||||
use zstd::DEFAULT_COMPRESSION_LEVEL;
|
||||
|
||||
use crate::{
|
||||
BaseVecIterator, CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error,
|
||||
GenericVec, RawVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
|
||||
CompressedPageMetadata, CompressedPagesMetadata, Error, GenericStoredVec, RawVec, Result,
|
||||
StoredIndex, StoredType, UnsafeSlice, Value, Version,
|
||||
};
|
||||
|
||||
const ONE_KIB: usize = 1024;
|
||||
@@ -152,14 +154,11 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> DynamicVec for CompressedVec<I, T>
|
||||
impl<I, T> GenericStoredVec<I, T> for CompressedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type I = I;
|
||||
type T = T;
|
||||
|
||||
#[inline]
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
|
||||
let page_index = Self::index_to_page_index(index);
|
||||
@@ -194,28 +193,6 @@ where
|
||||
fn path(&self) -> &Path {
|
||||
self.inner.path()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> GenericVec<I, T> for CompressedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
|
||||
let stored_len = self.stored_len();
|
||||
let from = from.unwrap_or_default();
|
||||
let to = to.map_or(stored_len, |i| i.min(stored_len));
|
||||
|
||||
if from >= stored_len || from >= to {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.iter_at_(from)
|
||||
.take(to - from)
|
||||
.map(|(_, v)| v.into_inner())
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
let pushed_len = self.pushed_len();
|
||||
@@ -352,11 +329,37 @@ where
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for CompressedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
self.inner.version()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn name(&self) -> String {
|
||||
self.name_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.len_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
self.modified_time_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> Clone for CompressedVec<I, T> {
|
||||
@@ -474,3 +477,30 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyIterableVec<I, T> for CompressedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyCollectableVec for CompressedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
use brk_exit::Exit;
|
||||
use clap_derive::ValueEnum;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use brk_core::StoredPhantom;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
|
||||
BoxedVecIterator, CollectableVec, Compressed, Result, StoredIndex, StoredType, Value, Version,
|
||||
};
|
||||
|
||||
use super::{
|
||||
ComputeFrom1, ComputeFrom2, ComputeFrom3, EagerVec, LazyVecFrom1, LazyVecFrom1Iterator,
|
||||
LazyVecFrom2, LazyVecFrom2Iterator, LazyVecFrom3, LazyVecFrom3Iterator, StoredVecIterator,
|
||||
};
|
||||
|
||||
#[derive(
|
||||
Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum,
|
||||
)]
|
||||
pub enum Computation {
|
||||
Eager,
|
||||
#[default]
|
||||
Lazy,
|
||||
}
|
||||
|
||||
impl Computation {
|
||||
pub fn eager(&self) -> bool {
|
||||
*self == Self::Eager
|
||||
}
|
||||
|
||||
pub fn lazy(&self) -> bool {
|
||||
*self == Self::Lazy
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T> {
|
||||
From1(BoxedAnyIterableVec<S1I, S1T>, ComputeFrom1<T, S1I, S1T>),
|
||||
From2(
|
||||
(BoxedAnyIterableVec<S1I, S1T>, BoxedAnyIterableVec<S2I, S2T>),
|
||||
ComputeFrom2<T, S1I, S1T, S2I, S2T>,
|
||||
),
|
||||
From3(
|
||||
(
|
||||
BoxedAnyIterableVec<S1I, S1T>,
|
||||
BoxedAnyIterableVec<S2I, S2T>,
|
||||
BoxedAnyIterableVec<S3I, S3T>,
|
||||
),
|
||||
ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
),
|
||||
}
|
||||
|
||||
pub type ComputedVecFrom1<I, T, S1I, S1T> =
|
||||
ComputedVec<I, T, S1I, S1T, StoredPhantom, StoredPhantom, StoredPhantom, StoredPhantom>;
|
||||
pub type ComputedVecFrom2<I, T, S1I, S1T, S2I, S2T> =
|
||||
ComputedVec<I, T, S1I, S1T, S2I, S2T, StoredPhantom, StoredPhantom>;
|
||||
pub type ComputedVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> =
|
||||
ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
|
||||
Eager {
|
||||
vec: EagerVec<I, T>,
|
||||
deps: Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
},
|
||||
LazyFrom1(LazyVecFrom1<I, T, S1I, S1T>),
|
||||
LazyFrom2(LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>),
|
||||
LazyFrom3(LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>),
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
pub fn forced_import_or_init_from_1(
|
||||
mode: Computation,
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
source: BoxedAnyIterableVec<S1I, S1T>,
|
||||
compute: ComputeFrom1<T, S1I, S1T>,
|
||||
) -> Result<Self> {
|
||||
Ok(match mode {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, version, compressed)?,
|
||||
deps: Dependencies::From1(source, compute),
|
||||
},
|
||||
Computation::Lazy => {
|
||||
Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import_or_init_from_2(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
mode: Computation,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
|
||||
) -> Result<Self> {
|
||||
Ok(match mode {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, version, compressed)?,
|
||||
deps: Dependencies::From2((source1, source2), compute),
|
||||
},
|
||||
Computation::Lazy => {
|
||||
Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import_or_init_from_3(
|
||||
mode: Computation,
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
compressed: Compressed,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
source3: BoxedAnyIterableVec<S3I, S3T>,
|
||||
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
) -> Result<Self> {
|
||||
Ok(match mode {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, version, compressed)?,
|
||||
deps: Dependencies::From3((source1, source2, source3), compute),
|
||||
},
|
||||
Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init(
|
||||
name, version, source1, source2, source3, compute,
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_if_necessary(&mut self, max_from: I, exit: &Exit) -> Result<()> {
|
||||
let (vec, dependencies) = if let ComputedVec::Eager {
|
||||
vec,
|
||||
deps: dependencies,
|
||||
} = self
|
||||
{
|
||||
(vec, dependencies)
|
||||
} else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
match dependencies {
|
||||
Dependencies::From1(source, compute) => {
|
||||
let version = source.version();
|
||||
let mut iter = source.iter();
|
||||
let t = |i: I| {
|
||||
compute(i.unwrap_to_usize(), &mut *iter)
|
||||
.map(|v| (i, v))
|
||||
.unwrap()
|
||||
};
|
||||
vec.compute_to(max_from, 1, version, t, exit)
|
||||
}
|
||||
Dependencies::From2((source1, source2), compute) => {
|
||||
let version = source1.version() + source2.version();
|
||||
let mut iter1 = source1.iter();
|
||||
let mut iter2 = source2.iter();
|
||||
let t = |i: I| {
|
||||
compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2)
|
||||
.map(|v| (i, v))
|
||||
.unwrap()
|
||||
};
|
||||
vec.compute_to(max_from, 1, version, t, exit)
|
||||
}
|
||||
Dependencies::From3((source1, source2, source3), compute) => {
|
||||
let version = source1.version() + source2.version() + source3.version();
|
||||
let mut iter1 = source1.iter();
|
||||
let mut iter2 = source2.iter();
|
||||
let mut iter3 = source3.iter();
|
||||
let t = |i: I| {
|
||||
compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2, &mut *iter3)
|
||||
.map(|v| (i, v))
|
||||
.unwrap()
|
||||
};
|
||||
vec.compute_to(max_from, 1, version, t, exit)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn version(&self) -> Version {
|
||||
match self {
|
||||
ComputedVec::Eager { vec, .. } => vec.version(),
|
||||
ComputedVec::LazyFrom1(v) => v.version(),
|
||||
ComputedVec::LazyFrom2(v) => v.version(),
|
||||
ComputedVec::LazyFrom3(v) => v.version(),
|
||||
}
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
match self {
|
||||
ComputedVec::Eager { vec, .. } => vec.name(),
|
||||
ComputedVec::LazyFrom1(v) => v.name(),
|
||||
ComputedVec::LazyFrom2(v) => v.name(),
|
||||
ComputedVec::LazyFrom3(v) => v.name(),
|
||||
}
|
||||
}
|
||||
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
ComputedVec::Eager { vec, .. } => vec.len(),
|
||||
ComputedVec::LazyFrom1(v) => v.len(),
|
||||
ComputedVec::LazyFrom2(v) => v.len(),
|
||||
ComputedVec::LazyFrom3(v) => v.len(),
|
||||
}
|
||||
}
|
||||
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
match self {
|
||||
ComputedVec::Eager { vec, .. } => vec.modified_time(),
|
||||
ComputedVec::LazyFrom1(v) => v.modified_time(),
|
||||
ComputedVec::LazyFrom2(v) => v.modified_time(),
|
||||
ComputedVec::LazyFrom3(v) => v.modified_time(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
|
||||
Eager(StoredVecIterator<'a, I, T>),
|
||||
LazyFrom1(LazyVecFrom1Iterator<'a, I, T, S1I, S1T>),
|
||||
LazyFrom2(LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>),
|
||||
LazyFrom3(LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>),
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator
|
||||
for ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self {
|
||||
Self::Eager(i) => i.next(),
|
||||
Self::LazyFrom1(i) => i.next(),
|
||||
Self::LazyFrom2(i) => i.next(),
|
||||
Self::LazyFrom3(i) => i.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> BaseVecIterator
|
||||
for ComputedVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
match self {
|
||||
Self::Eager(i) => i.mut_index(),
|
||||
Self::LazyFrom1(i) => i.mut_index(),
|
||||
Self::LazyFrom2(i) => i.mut_index(),
|
||||
Self::LazyFrom3(i) => i.mut_index(),
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Eager(i) => i.len(),
|
||||
Self::LazyFrom1(i) => i.len(),
|
||||
Self::LazyFrom2(i) => i.len(),
|
||||
Self::LazyFrom3(i) => i.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator
|
||||
for &'a ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
match self {
|
||||
ComputedVec::Eager { vec, .. } => ComputedVecIterator::Eager(vec.into_iter()),
|
||||
ComputedVec::LazyFrom1(v) => ComputedVecIterator::LazyFrom1(v.into_iter()),
|
||||
ComputedVec::LazyFrom2(v) => ComputedVecIterator::LazyFrom2(v.into_iter()),
|
||||
ComputedVec::LazyFrom3(v) => ComputedVecIterator::LazyFrom3(v.into_iter()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>
|
||||
for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyCollectableVec
|
||||
for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
use core::error;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt::Debug,
|
||||
ops::Add,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex};
|
||||
use brk_exit::Exit;
|
||||
use log::info;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
|
||||
GenericStoredVec, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value,
|
||||
VecIterator, Version,
|
||||
};
|
||||
|
||||
const ONE_KIB: usize = 1024;
|
||||
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
|
||||
const MAX_CACHE_SIZE: usize = 210 * ONE_MIB;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EagerVec<I, T> {
|
||||
computed_version: Option<Version>,
|
||||
inner: StoredVec<I, T>,
|
||||
}
|
||||
|
||||
impl<I, T> EagerVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
const SIZE_OF: usize = size_of::<T>();
|
||||
|
||||
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
|
||||
let inner = StoredVec::forced_import(path, version, compressed)?;
|
||||
|
||||
Ok(Self {
|
||||
computed_version: None,
|
||||
inner,
|
||||
})
|
||||
}
|
||||
|
||||
fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> {
|
||||
if exit.triggered() {
|
||||
return Ok(());
|
||||
}
|
||||
exit.block();
|
||||
self.inner.truncate_if_needed(index)?;
|
||||
exit.release();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
|
||||
match self.len().cmp(&index.to_usize()?) {
|
||||
Ordering::Less => {
|
||||
return Err(Error::IndexTooHigh);
|
||||
}
|
||||
ord => {
|
||||
if ord == Ordering::Greater {
|
||||
self.safe_truncate_if_needed(index, exit)?;
|
||||
}
|
||||
self.inner.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.inner.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE {
|
||||
self.safe_flush(exit)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
if exit.triggered() {
|
||||
return Ok(());
|
||||
}
|
||||
exit.block();
|
||||
self.inner.flush()?;
|
||||
exit.release();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
self.computed_version.unwrap()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.is_empty()
|
||||
}
|
||||
|
||||
pub fn name(&self) -> String {
|
||||
self.inner.name()
|
||||
}
|
||||
|
||||
pub fn modified_time(&self) -> Result<Duration> {
|
||||
self.inner.modified_time()
|
||||
}
|
||||
|
||||
// pub fn vec(&self) -> &StoredVec<I, T> {
|
||||
// &self.inner
|
||||
// }
|
||||
|
||||
// pub fn mut_vec(&mut self) -> &StoredVec<I, T> {
|
||||
// &mut self.inner
|
||||
// }
|
||||
|
||||
// pub fn any_vec(&self) -> &dyn AnyVec {
|
||||
// &self.inner
|
||||
// }
|
||||
|
||||
// pub fn mut_any_vec(&mut self) -> &mut dyn AnyVec {
|
||||
// &mut self.inner
|
||||
// }
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
self.inner.path()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path_computed_version(&self) -> PathBuf {
|
||||
self.inner.path().join("computed_version")
|
||||
}
|
||||
|
||||
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
let path = self.path_computed_version();
|
||||
if version.validate(path.as_ref()).is_err() {
|
||||
self.inner.reset()?;
|
||||
}
|
||||
version.write(path.as_ref())?;
|
||||
|
||||
if self.is_empty() {
|
||||
info!("Computing {}...", self.name())
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> StoredVecIterator<I, T> {
|
||||
self.into_iter()
|
||||
}
|
||||
|
||||
pub fn compute_to<F>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
to: usize,
|
||||
version: Version,
|
||||
mut t: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
F: FnMut(I) -> (I, T),
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
(index.to_usize()?..to).try_for_each(|i| {
|
||||
let (i, v) = t(I::from(i));
|
||||
self.forced_push_at(i, v, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_range<A, F>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
other: &impl AnyIterableVec<I, A>,
|
||||
t: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
A: StoredType,
|
||||
F: FnMut(I) -> (I, T),
|
||||
{
|
||||
self.compute_to(max_from, other.len(), other.version(), t, exit)
|
||||
}
|
||||
|
||||
pub fn compute_transform<A, B, F>(
|
||||
&mut self,
|
||||
max_from: A,
|
||||
other: &impl AnyIterableVec<A, B>,
|
||||
mut t: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
A: StoredIndex,
|
||||
B: StoredType,
|
||||
F: FnMut((A, B, &Self)) -> (I, T),
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + other.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(A::from(self.len()));
|
||||
other.iter_at(index).try_for_each(|(a, b)| {
|
||||
let (i, v) = t((a, b.into_inner(), self));
|
||||
self.forced_push_at(i, v, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_more_to_less(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
other: &impl AnyIterableVec<T, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType + StoredIndex,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + other.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(
|
||||
VecIterator::last(self.inner.into_iter())
|
||||
.map_or_else(T::default, |(_, v)| v.into_inner()),
|
||||
);
|
||||
let mut prev_i = None;
|
||||
other.iter_at(index).try_for_each(|(v, i)| -> Result<()> {
|
||||
let i = i.into_inner();
|
||||
if prev_i.is_some_and(|prev_i| prev_i == i) {
|
||||
return Ok(());
|
||||
}
|
||||
if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) {
|
||||
self.forced_push_at(i, v, exit)?;
|
||||
}
|
||||
prev_i.replace(i);
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_less_to_more(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
first_indexes: &impl AnyIterableVec<T, I>,
|
||||
indexes_count: &impl AnyIterableVec<T, StoredUsize>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO
|
||||
+ self.inner.version()
|
||||
+ first_indexes.version()
|
||||
+ indexes_count.version(),
|
||||
)?;
|
||||
|
||||
let mut indexes_count_iter = indexes_count.iter();
|
||||
|
||||
let index = max_from.min(T::from(self.len()));
|
||||
first_indexes
|
||||
.iter_at(index)
|
||||
.try_for_each(|(value, first_index)| {
|
||||
let first_index = (first_index).to_usize()?;
|
||||
let count = *indexes_count_iter.unwrap_get_inner(value);
|
||||
(first_index..first_index + count)
|
||||
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_count_from_indexes<T2, T3>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &impl AnyIterableVec<I, T2>,
|
||||
other_to_else: &impl AnyIterableVec<T2, T3>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType
|
||||
+ StoredIndex
|
||||
+ Copy
|
||||
+ Add<usize, Output = T2>
|
||||
+ CheckedSub<T2>
|
||||
+ TryInto<T>
|
||||
+ Default,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
T3: StoredType,
|
||||
{
|
||||
let opt: Option<Box<dyn FnMut(T2) -> bool>> = None;
|
||||
self.compute_filtered_count_from_indexes_(max_from, first_indexes, other_to_else, opt, exit)
|
||||
}
|
||||
|
||||
pub fn compute_filtered_count_from_indexes<T2, T3, F>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &impl AnyIterableVec<I, T2>,
|
||||
other_to_else: &impl AnyIterableVec<T2, T3>,
|
||||
filter: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType
|
||||
+ StoredIndex
|
||||
+ Copy
|
||||
+ Add<usize, Output = T2>
|
||||
+ CheckedSub<T2>
|
||||
+ TryInto<T>
|
||||
+ Default,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
T3: StoredType,
|
||||
F: FnMut(T2) -> bool,
|
||||
{
|
||||
self.compute_filtered_count_from_indexes_(
|
||||
max_from,
|
||||
first_indexes,
|
||||
other_to_else,
|
||||
Some(Box::new(filter)),
|
||||
exit,
|
||||
)
|
||||
}
|
||||
|
||||
fn compute_filtered_count_from_indexes_<T2, T3>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &impl AnyIterableVec<I, T2>,
|
||||
other_to_else: &impl AnyIterableVec<T2, T3>,
|
||||
mut filter: Option<Box<dyn FnMut(T2) -> bool + '_>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType
|
||||
+ StoredIndex
|
||||
+ Copy
|
||||
+ Add<usize, Output = T2>
|
||||
+ CheckedSub<T2>
|
||||
+ TryInto<T>
|
||||
+ Default,
|
||||
T3: StoredType,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + first_indexes.version(),
|
||||
)?;
|
||||
|
||||
let mut other_iter = first_indexes.iter();
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes
|
||||
.iter_at(index)
|
||||
.try_for_each(|(i, first_index)| {
|
||||
let end = other_iter
|
||||
.get_inner(i + 1)
|
||||
.map(|v| v.unwrap_to_usize())
|
||||
.unwrap_or_else(|| other_to_else.len());
|
||||
|
||||
let range = first_index.unwrap_to_usize()..end;
|
||||
let count = if let Some(filter) = filter.as_mut() {
|
||||
range.into_iter().filter(|i| filter(T2::from(*i))).count()
|
||||
} else {
|
||||
range.count()
|
||||
};
|
||||
self.forced_push_at(i, T::from(T2::from(count)), exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_is_first_ordered<A>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
self_to_other: &impl AnyIterableVec<I, A>,
|
||||
other_to_self: &impl AnyIterableVec<A, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: From<bool>,
|
||||
A: StoredIndex + StoredType,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO
|
||||
+ self.inner.version()
|
||||
+ self_to_other.version()
|
||||
+ other_to_self.version(),
|
||||
)?;
|
||||
|
||||
let mut other_to_self_iter = other_to_self.iter();
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
self_to_other.iter_at(index).try_for_each(|(i, other)| {
|
||||
self.forced_push_at(
|
||||
i,
|
||||
T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i),
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
|
||||
pub fn compute_sum_from_indexes<T2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &impl AnyIterableVec<I, T2>,
|
||||
indexes_count: &impl AnyIterableVec<I, StoredUsize>,
|
||||
source: &impl AnyIterableVec<T2, T>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<usize> + Add<T, Output = T>,
|
||||
T2: StoredIndex + StoredType,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO
|
||||
+ self.inner.version()
|
||||
+ first_indexes.version()
|
||||
+ indexes_count.version(),
|
||||
)?;
|
||||
|
||||
let mut indexes_count_iter = indexes_count.iter();
|
||||
let mut source_iter = source.iter();
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes
|
||||
.iter_at(index)
|
||||
.try_for_each(|(i, first_index)| {
|
||||
let count = *indexes_count_iter.unwrap_get_inner(i);
|
||||
let first_index = first_index.unwrap_to_usize();
|
||||
let range = first_index..first_index + count;
|
||||
let mut sum = T::from(0_usize);
|
||||
range.into_iter().for_each(|i| {
|
||||
sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i));
|
||||
});
|
||||
self.forced_push_at(i, sum, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I> EagerVec<I, Bitcoin>
|
||||
where
|
||||
I: StoredIndex,
|
||||
{
|
||||
pub fn compute_from_sats(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
sats: &impl AnyIterableVec<I, Sats>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + sats.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
sats.iter_at(index).try_for_each(|(i, sats)| {
|
||||
let (i, v) = (i, Bitcoin::from(sats.into_inner()));
|
||||
self.forced_push_at(i, v, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl EagerVec<Height, Dollars> {
|
||||
pub fn compute_from_bitcoin(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
bitcoin: &impl AnyIterableVec<Height, Bitcoin>,
|
||||
price: &impl AnyIterableVec<Height, Close<Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + bitcoin.version(),
|
||||
)?;
|
||||
|
||||
let mut price_iter = price.iter();
|
||||
let index = max_from.min(Height::from(self.len()));
|
||||
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
|
||||
let dollars = price_iter.unwrap_get_inner(i);
|
||||
let (i, v) = (i, *dollars * bitcoin.into_inner());
|
||||
self.forced_push_at(i, v, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl EagerVec<TxIndex, Dollars> {
|
||||
pub fn compute_from_bitcoin(
|
||||
&mut self,
|
||||
max_from: TxIndex,
|
||||
bitcoin: &impl AnyIterableVec<TxIndex, Bitcoin>,
|
||||
i_to_height: &impl AnyIterableVec<TxIndex, Height>,
|
||||
price: &impl AnyIterableVec<Height, Close<Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + bitcoin.version(),
|
||||
)?;
|
||||
|
||||
let mut i_to_height_iter = i_to_height.iter();
|
||||
let mut price_iter = price.iter();
|
||||
let index = max_from.min(TxIndex::from(self.len()));
|
||||
bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| {
|
||||
let height = i_to_height_iter.unwrap_get_inner(i);
|
||||
let dollars = price_iter.unwrap_get_inner(height);
|
||||
let (i, v) = (i, *dollars * bitcoin.into_inner());
|
||||
self.forced_push_at(i, v, exit)
|
||||
})?;
|
||||
|
||||
self.safe_flush(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T> IntoIterator for &'a EagerVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = StoredVecIterator<'a, I, T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.inner.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for EagerVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
self.inner.version()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn name(&self) -> String {
|
||||
self.inner.name()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
self.inner.modified_time()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyIterableVec<I, T> for EagerVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
{
|
||||
Box::new(self.inner.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyCollectableVec for EagerVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt::Debug,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use brk_core::Height;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
|
||||
GenericStoredVec, Mmap, Result, StoredIndex, StoredType, StoredVec, Value, Version,
|
||||
};
|
||||
|
||||
use super::StoredVecIterator;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IndexedVec<I, T> {
|
||||
height: Option<Height>,
|
||||
inner: StoredVec<I, T>,
|
||||
}
|
||||
|
||||
impl<I, T> IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
|
||||
Ok(Self {
|
||||
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
|
||||
inner: StoredVec::forced_import(path, version, compressed)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
|
||||
self.inner.get_or_read(index, mmap)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
|
||||
let len = self.inner.len();
|
||||
match len.cmp(&index.to_usize()?) {
|
||||
Ordering::Greater => {
|
||||
// dbg!(len, index, &self.pathbuf);
|
||||
// panic!();
|
||||
Ok(())
|
||||
}
|
||||
Ordering::Equal => {
|
||||
self.inner.push(value);
|
||||
Ok(())
|
||||
}
|
||||
Ordering::Less => {
|
||||
dbg!(index, value, len, self.path_height());
|
||||
Err(Error::IndexTooHigh)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> {
|
||||
if self.height.is_none_or(|self_height| self_height != height) {
|
||||
height.write(&self.path_height())?;
|
||||
}
|
||||
self.inner.truncate_if_needed(index)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn flush(&mut self, height: Height) -> Result<()> {
|
||||
height.write(&self.path_height())?;
|
||||
self.inner.flush()
|
||||
}
|
||||
|
||||
pub fn vec(&self) -> &StoredVec<I, T> {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
pub fn mut_vec(&mut self) -> &mut StoredVec<I, T> {
|
||||
&mut self.inner
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn hasnt(&self, index: I) -> Result<bool> {
|
||||
self.inner.has(index).map(|b| !b)
|
||||
}
|
||||
|
||||
pub fn height(&self) -> brk_core::Result<Height> {
|
||||
Height::try_from(self.path_height().as_path())
|
||||
}
|
||||
fn path_height(&self) -> PathBuf {
|
||||
Self::path_height_(self.inner.path())
|
||||
}
|
||||
fn path_height_(path: &Path) -> PathBuf {
|
||||
path.join("height")
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
self.inner.version()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn name(&self) -> String {
|
||||
self.inner.name()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
self.inner.modified_time()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyIndexedVec: AnyVec {
|
||||
fn height(&self) -> brk_core::Result<Height>;
|
||||
fn flush(&mut self, height: Height) -> Result<()>;
|
||||
}
|
||||
|
||||
impl<I, T> AnyIndexedVec for IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn height(&self) -> brk_core::Result<Height> {
|
||||
self.height()
|
||||
}
|
||||
|
||||
fn flush(&mut self, height: Height) -> Result<()> {
|
||||
self.flush(height)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T> IntoIterator for &'a IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = StoredVecIterator<'a, I, T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.inner.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyIterableVec<I, T> for IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyCollectableVec for IndexedVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
|
||||
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
|
||||
};
|
||||
|
||||
pub type ComputeFrom1<T, S1I, S1T> =
|
||||
for<'a> fn(usize, &mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>) -> Option<T>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LazyVecFrom1<I, T, S1I, S1T> {
|
||||
name: String,
|
||||
version: Version,
|
||||
source: BoxedAnyIterableVec<S1I, S1T>,
|
||||
compute: ComputeFrom1<T, S1I, S1T>,
|
||||
phantom: PhantomData<I>,
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T> LazyVecFrom1<I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
pub fn init(
|
||||
name: &str,
|
||||
version: Version,
|
||||
source: BoxedAnyIterableVec<S1I, S1T>,
|
||||
compute: ComputeFrom1<T, S1I, S1T>,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.to_owned(),
|
||||
version,
|
||||
source,
|
||||
compute,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> {
|
||||
lazy: &'a LazyVecFrom1<I, T, S1I, S1T>,
|
||||
source: BoxedVecIterator<'a, S1I, S1T>,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T> Iterator for LazyVecFrom1Iterator<'a, I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let opt = (self.lazy.compute)(self.index, &mut *self.source)
|
||||
.map(|v| (I::from(self.index), Value::Owned(v)));
|
||||
if opt.is_some() {
|
||||
self.index += 1;
|
||||
}
|
||||
opt
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T> BaseVecIterator for LazyVecFrom1Iterator<'_, I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
&mut self.index
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.source.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T> IntoIterator for &'a LazyVecFrom1<I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
LazyVecFrom1Iterator {
|
||||
lazy: self,
|
||||
source: self.source.iter(),
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T> AnyVec for LazyVecFrom1<I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
fn version(&self) -> Version {
|
||||
self.version()
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
}
|
||||
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.source.len()
|
||||
}
|
||||
|
||||
fn modified_time(&self) -> Result<std::time::Duration> {
|
||||
self.source.modified_time()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T> AnyIterableVec<I, T> for LazyVecFrom1<I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T> AnyCollectableVec for LazyVecFrom1<I, T, S1I, S1T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
|
||||
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
|
||||
};
|
||||
|
||||
pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
|
||||
usize,
|
||||
&mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>,
|
||||
&mut dyn BaseVecIterator<Item = (S2I, Value<'a, S2T>)>,
|
||||
) -> Option<T>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LazyVecFrom2<I, T, S1I, S1T, S2I, S2T> {
|
||||
name: String,
|
||||
version: Version,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
|
||||
phantom: PhantomData<I>,
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T> LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
pub fn init(
|
||||
name: &str,
|
||||
version: Version,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
version,
|
||||
source1,
|
||||
source2,
|
||||
compute,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> {
|
||||
lazy: &'a LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>,
|
||||
source1: BoxedVecIterator<'a, S1I, S1T>,
|
||||
source2: BoxedVecIterator<'a, S2I, S2T>,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let opt = (self.lazy.compute)(self.index, &mut *self.source1, &mut *self.source2)
|
||||
.map(|v| (I::from(self.index), Value::Owned(v)));
|
||||
if opt.is_some() {
|
||||
self.index += 1;
|
||||
}
|
||||
opt
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T> BaseVecIterator
|
||||
for LazyVecFrom2Iterator<'_, I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
&mut self.index
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.source1.len().min(self.source2.len())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
LazyVecFrom2Iterator {
|
||||
lazy: self,
|
||||
source1: self.source1.iter(),
|
||||
source2: self.source2.iter(),
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T> AnyVec for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
fn version(&self) -> Version {
|
||||
self.version()
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
}
|
||||
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.source1.len().min(self.source2.len())
|
||||
}
|
||||
|
||||
fn modified_time(&self) -> Result<std::time::Duration> {
|
||||
Ok(self
|
||||
.source1
|
||||
.modified_time()?
|
||||
.min(self.source2.modified_time()?))
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T> AnyIterableVec<I, T> for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T> AnyCollectableVec for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
|
||||
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
|
||||
};
|
||||
|
||||
pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
|
||||
usize,
|
||||
&mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>,
|
||||
&mut dyn BaseVecIterator<Item = (S2I, Value<'a, S2T>)>,
|
||||
&mut dyn BaseVecIterator<Item = (S3I, Value<'a, S3T>)>,
|
||||
) -> Option<T>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
|
||||
name: String,
|
||||
version: Version,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
source3: BoxedAnyIterableVec<S3I, S3T>,
|
||||
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
phantom: PhantomData<I>,
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
pub fn init(
|
||||
name: &str,
|
||||
version: Version,
|
||||
source1: BoxedAnyIterableVec<S1I, S1T>,
|
||||
source2: BoxedAnyIterableVec<S2I, S2T>,
|
||||
source3: BoxedAnyIterableVec<S3I, S3T>,
|
||||
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
version,
|
||||
source1,
|
||||
source2,
|
||||
source3,
|
||||
compute,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
|
||||
lazy: &'a LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>,
|
||||
source1: BoxedVecIterator<'a, S1I, S1T>,
|
||||
source2: BoxedVecIterator<'a, S2I, S2T>,
|
||||
source3: BoxedVecIterator<'a, S3I, S3T>,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator
|
||||
for LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let opt = (self.lazy.compute)(
|
||||
self.index,
|
||||
&mut *self.source1,
|
||||
&mut *self.source2,
|
||||
&mut *self.source3,
|
||||
)
|
||||
.map(|v| (I::from(self.index), Value::Owned(v)));
|
||||
if opt.is_some() {
|
||||
self.index += 1;
|
||||
}
|
||||
opt
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> BaseVecIterator
|
||||
for LazyVecFrom3Iterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
&mut self.index
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.source1
|
||||
.len()
|
||||
.min(self.source2.len())
|
||||
.min(self.source3.len())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator
|
||||
for &'a LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType + 'a,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
LazyVecFrom3Iterator {
|
||||
lazy: self,
|
||||
source1: self.source1.iter(),
|
||||
source2: self.source2.iter(),
|
||||
source3: self.source3.iter(),
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn version(&self) -> Version {
|
||||
self.version()
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
}
|
||||
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.source1
|
||||
.len()
|
||||
.min(self.source2.len())
|
||||
.min(self.source3.len())
|
||||
}
|
||||
|
||||
fn modified_time(&self) -> Result<std::time::Duration> {
|
||||
Ok(self
|
||||
.source1
|
||||
.modified_time()?
|
||||
.min(self.source2.modified_time()?)
|
||||
.min(self.source3.modified_time()?))
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>
|
||||
for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyCollectableVec
|
||||
for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
S1I: StoredIndex,
|
||||
S1T: StoredType,
|
||||
S2I: StoredIndex,
|
||||
S2T: StoredType,
|
||||
S3I: StoredIndex,
|
||||
S3T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,19 @@
|
||||
mod compressed;
|
||||
mod computed;
|
||||
mod eager;
|
||||
mod indexed;
|
||||
mod lazy1;
|
||||
mod lazy2;
|
||||
mod lazy3;
|
||||
mod raw;
|
||||
mod stored;
|
||||
|
||||
pub use compressed::*;
|
||||
pub use computed::*;
|
||||
pub use eager::*;
|
||||
pub use indexed::*;
|
||||
pub use lazy1::*;
|
||||
pub use lazy2::*;
|
||||
pub use lazy3::*;
|
||||
pub use raw::*;
|
||||
pub use stored::*;
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
mem,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use arc_swap::{ArcSwap, Guard};
|
||||
@@ -11,8 +12,8 @@ use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
BaseVecIterator, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice,
|
||||
Value, Version,
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
|
||||
Error, GenericStoredVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -79,19 +80,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> DynamicVec for RawVec<I, T>
|
||||
impl<I, T> GenericStoredVec<I, T> for RawVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type I = I;
|
||||
type T = T;
|
||||
|
||||
#[inline]
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
|
||||
let index = index * Self::SIZE_OF_T;
|
||||
let slice = &mmap[index..(index + Self::SIZE_OF_T)];
|
||||
Self::T::try_read_from_bytes(slice)
|
||||
T::try_read_from_bytes(slice)
|
||||
.map(|v| Some(v))
|
||||
.map_err(Error::from)
|
||||
}
|
||||
@@ -119,28 +117,6 @@ where
|
||||
fn path(&self) -> &Path {
|
||||
self.pathbuf.as_path()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> GenericVec<I, T> for RawVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
|
||||
let stored_len = self.stored_len();
|
||||
let from = from.unwrap_or_default();
|
||||
let to = to.map_or(stored_len, |i| i.min(stored_len));
|
||||
|
||||
if from >= stored_len || from >= to {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.iter_at_(from)
|
||||
.take(to - from)
|
||||
.map(|(_, v)| v.into_inner())
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
let pushed_len = self.pushed_len();
|
||||
@@ -188,10 +164,41 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// #[inline]
|
||||
// fn version(&self) -> Version {
|
||||
// self.version
|
||||
// }
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for RawVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn name(&self) -> String {
|
||||
self.name_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.len_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
self.modified_time_()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> Clone for RawVec<I, T> {
|
||||
@@ -270,3 +277,30 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyIterableVec<I, T> for RawVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyCollectableVec for RawVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
|
||||
Compressed, GenericStoredVec, Result, StoredIndex, StoredType, Value, Version,
|
||||
};
|
||||
|
||||
use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum StoredVec<I, T> {
|
||||
Raw(RawVec<I, T>),
|
||||
Compressed(CompressedVec<I, T>),
|
||||
}
|
||||
|
||||
impl<I, T> StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
|
||||
if *compressed {
|
||||
Ok(Self::Compressed(CompressedVec::forced_import(
|
||||
path, version,
|
||||
)?))
|
||||
} else {
|
||||
Ok(Self::Raw(RawVec::forced_import(path, version)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> GenericStoredVec<I, T> for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.read_(index, guard),
|
||||
StoredVec::Compressed(v) => v.read_(index, guard),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mmap(&self) -> &ArcSwap<Mmap> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.mmap(),
|
||||
StoredVec::Compressed(v) => v.mmap(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn stored_len(&self) -> usize {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.stored_len(),
|
||||
StoredVec::Compressed(v) => v.stored_len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn pushed(&self) -> &[T] {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.pushed(),
|
||||
StoredVec::Compressed(v) => v.pushed(),
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn mut_pushed(&mut self) -> &mut Vec<T> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.mut_pushed(),
|
||||
StoredVec::Compressed(v) => v.mut_pushed(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path(&self) -> &Path {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.path(),
|
||||
StoredVec::Compressed(v) => v.path(),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.flush(),
|
||||
StoredVec::Compressed(v) => v.flush(),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.truncate_if_needed(index),
|
||||
StoredVec::Compressed(v) => v.truncate_if_needed(index),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyVec for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn version(&self) -> Version {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.version(),
|
||||
StoredVec::Compressed(v) => v.version(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_type_to_string(&self) -> &str {
|
||||
I::to_string()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.pushed_len() + self.stored_len()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn modified_time(&self) -> Result<Duration> {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.modified_time(),
|
||||
StoredVec::Compressed(v) => v.modified_time(),
|
||||
}
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.name(),
|
||||
StoredVec::Compressed(v) => v.name(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum StoredVecIterator<'a, I, T> {
|
||||
Raw(RawVecIterator<'a, I, T>),
|
||||
Compressed(CompressedVecIterator<'a, I, T>),
|
||||
}
|
||||
|
||||
impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self {
|
||||
Self::Compressed(i) => i.next(),
|
||||
Self::Raw(i) => i.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> BaseVecIterator for StoredVecIterator<'_, I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
#[inline]
|
||||
fn mut_index(&mut self) -> &mut usize {
|
||||
match self {
|
||||
Self::Compressed(iter) => iter.mut_index(),
|
||||
Self::Raw(iter) => iter.mut_index(),
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Compressed(i) => i.len(),
|
||||
Self::Raw(i) => i.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, T> IntoIterator for &'a StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
type Item = (I, Value<'a, T>);
|
||||
type IntoIter = StoredVecIterator<'a, I, T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
match self {
|
||||
StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()),
|
||||
StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyIterableVec<I, T> for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
|
||||
where
|
||||
T: 'a,
|
||||
{
|
||||
Box::new(self.into_iter())
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> AnyCollectableVec for StoredVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn collect_range_serde_json(
|
||||
&self,
|
||||
from: Option<i64>,
|
||||
to: Option<i64>,
|
||||
) -> Result<Vec<serde_json::Value>> {
|
||||
CollectableVec::collect_range_serde_json(self, from, to)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user